This is an archive of the discontinued LLVM Phabricator instance.

Differential D137525

AMDGPU: Drop and upgrade llvm.amdgcn.atomic.inc/dec to atomicrmw
ClosedPublic

Authored by arsenm on Nov 6 2022, 9:13 PM.

Details

Reviewers
vangthao
pravinjagtap
Pierre-vh
cdevadas
rampitec
Joe_Nash
Group Reviewers
Restricted Project

Diff Detail

Event Timeline

arsenm created this revision.Nov 6 2022, 9:13 PM
Herald added a project: Restricted Project. · View Herald TranscriptNov 6 2022, 9:13 PM
Herald added subscribers: kosarev, foad, kerbowa and 6 others. · View Herald Transcript
arsenm requested review of this revision.Nov 6 2022, 9:13 PM
Herald added a project: Restricted Project. · View Herald TranscriptNov 6 2022, 9:13 PM
Herald added a subscriber: wdng. · View Herald Transcript
arsenm added parent revisions: D137524: clang/AMDGPU: Emit atomicrmw for atomic_inc/dec builtins, D137362: AMDGPU: Support atomicrmw uinc_wrap/udec_wrap.Nov 6 2022, 9:13 PM
Harbormaster completed remote builds in B196398: Diff 473553.Nov 6 2022, 9:14 PM
arsenm updated this revision to Diff 486620.Jan 5 2023, 10:21 AM

Rebase

Harbormaster completed remote builds in B205945: Diff 486620.Jan 5 2023, 10:22 AM
arsenm added a comment.Jun 9 2023, 6:58 PM

ping

arsenm added a reviewer: vangthao.Jun 9 2023, 7:00 PM
arsenm added a reviewer: pravinjagtap.Jun 14 2023, 6:34 AM
arsenm added a comment.Jun 21 2023, 11:16 AM

ping

arsenm added reviewers: Pierre-vh, cdevadas, rampitec, Joe_Nash.Jun 21 2023, 11:17 AM
cdevadas added inline comments.Jun 21 2023, 12:48 PM
llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
70

There are additional waitcnt instructions in most of the lit pattern changes. Were they regressions earlier?

arsenm added inline comments.Jun 21 2023, 12:51 PM
llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
70

Without an ordering passed through to the memory legalizer, these weren’t actually treated as atomics. They were not properly legalized before

cdevadas accepted this revision.Jun 21 2023, 12:56 PM

LGTM

This revision is now accepted and ready to land.Jun 21 2023, 12:56 PM
arsenm closed this revision.Jun 21 2023, 6:20 PM

92ee60b66f581fdd919315da5c6ae631e581b021

Revision Contents

PathSize
llvm/
include/
llvm/
IR/
15 lines
Support/
3 lines
lib/
IR/
50 lines
Target/
AMDGPU/
3 lines
18 lines
2 lines
6 lines
18 lines
test/
Analysis/
DivergenceAnalysis/
AMDGPU/
31 lines
LegacyDivergenceAnalysis/
AMDGPU/
31 lines
Bitcode/
115 lines
CodeGen/
AMDGPU/
GlobalISel/
2948 lines
atomicrmw_uinc_wrap.ll
llvm.amdgcn.atomic.inc.ll
1232 lines
50 lines
12 lines
37 lines
Transforms/
InferAddressSpaces/
AMDGPU/
113 lines
LoopStrengthReduce/
AMDGPU/
82 lines
Verifier/
AMDGPU/
38 lines

Diff 486620

llvm/include/llvm/IR/IntrinsicsAMDGPU.td

Show First 20 Lines • Show All 436 Lines • ▼ Show 20 Lines
// v_mad_f32|f16/v_mac_f32|f16, selected regardless of denorm support. // v_mad_f32|f16/v_mac_f32|f16, selected regardless of denorm support.
def int_amdgcn_fmad_ftz : def int_amdgcn_fmad_ftz :
DefaultAttrsIntrinsic<[llvm_anyfloat_ty], DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
[IntrNoMem, IntrSpeculatable] [IntrNoMem, IntrSpeculatable]
>; >;
// Fields should mirror atomicrmw
class AMDGPUAtomicIncIntrin : Intrinsic<[llvm_anyint_ty],
[llvm_anyptr_ty,
LLVMMatchType<0>,
llvm_i32_ty, // ordering
llvm_i32_ty, // scope
llvm_i1_ty], // isVolatile
[IntrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>,
ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, IntrNoCallback, IntrNoFree], "",
[SDNPMemOperand]
>;
def int_amdgcn_atomic_inc : AMDGPUAtomicIncIntrin;
def int_amdgcn_atomic_dec : AMDGPUAtomicIncIntrin;
class AMDGPULDSIntrin : class AMDGPULDSIntrin :
Intrinsic<[llvm_any_ty], Intrinsic<[llvm_any_ty],
[LLVMQualPointerType<LLVMMatchType<0>, 3>, [LLVMQualPointerType<LLVMMatchType<0>, 3>,
LLVMMatchType<0>, LLVMMatchType<0>,
llvm_i32_ty, // ordering llvm_i32_ty, // ordering
llvm_i32_ty, // scope llvm_i32_ty, // scope
llvm_i1_ty], // isVolatile llvm_i1_ty], // isVolatile
[IntrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>, [IntrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>,
▲ Show 20 Lines • Show All 2,005 Lines • Show Last 20 Lines

llvm/include/llvm/Support/AtomicOrdering.h

Show First 20 Lines • Show All 68 Lines • ▼ Show 20 Lines
bool operator>(AtomicOrdering, AtomicOrdering) = delete; bool operator>(AtomicOrdering, AtomicOrdering) = delete;
bool operator<=(AtomicOrdering, AtomicOrdering) = delete; bool operator<=(AtomicOrdering, AtomicOrdering) = delete;
bool operator>=(AtomicOrdering, AtomicOrdering) = delete; bool operator>=(AtomicOrdering, AtomicOrdering) = delete;
// Validate an integral value which isn't known to fit within the enum's range // Validate an integral value which isn't known to fit within the enum's range
// is a valid AtomicOrdering. // is a valid AtomicOrdering.
template <typename Int> inline bool isValidAtomicOrdering(Int I) { template <typename Int> inline bool isValidAtomicOrdering(Int I) {
return static_cast<Int>(AtomicOrdering::NotAtomic) <= I && return static_cast<Int>(AtomicOrdering::NotAtomic) <= I &&
I <= static_cast<Int>(AtomicOrdering::SequentiallyConsistent); I <= static_cast<Int>(AtomicOrdering::SequentiallyConsistent) &&
I != 3;
} }
/// String used by LLVM IR to represent atomic ordering. /// String used by LLVM IR to represent atomic ordering.
inline const char *toIRString(AtomicOrdering ao) { inline const char *toIRString(AtomicOrdering ao) {
static const char *names[8] = {"not_atomic", "unordered", "monotonic", static const char *names[8] = {"not_atomic", "unordered", "monotonic",
"consume", "acquire", "release", "consume", "acquire", "release",
"acq_rel", "seq_cst"}; "acq_rel", "seq_cst"};
return names[static_cast<size_t>(ao)]; return names[static_cast<size_t>(ao)];
▲ Show 20 Lines • Show All 77 Lines • Show Last 20 Lines

llvm/lib/IR/AutoUpgrade.cpp

Show First 20 Lines • Show All 808 Lines • ▼ Show 20 Lines if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" || Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" ||
Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" || Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" || Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" ||
Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" || Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" || Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" ||
Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1") Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1")
return true; return true;
if (Name == "amdgcn.alignbit") { if (Name.startswith("amdgcn."))
Name = Name.substr(7); // Strip off "amdgcn."
if (Name == "alignbit") {
// Target specific intrinsic became redundant // Target specific intrinsic became redundant
NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr, NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr,
{F->getReturnType()}); {F->getReturnType()});
return true; return true;
} }
break; if (Name.startswith("atomic.inc") || Name.startswith("atomic.dec")) {
// This was replaced with atomicrmw uinc_wrap and udec_wrap, so there's no
// new declaration.
NewFn = nullptr;
return true;
} }
break;
}
case 'c': { case 'c': {
if (Name.startswith("ctlz.") && F->arg_size() == 1) { if (Name.startswith("ctlz.") && F->arg_size() == 1) {
rename(F); rename(F);
NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
F->arg_begin()->getType()); F->arg_begin()->getType());
return true; return true;
} }
if (Name.startswith("cttz.") && F->arg_size() == 1) { if (Name.startswith("cttz.") && F->arg_size() == 1) {
▲ Show 20 Lines • Show All 1,221 Lines • ▼ Show 20 Lines if (Name == "mve.vctp64.old") {
} }
Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
return Builder.CreateCall(Fn, Ops, CI->getName()); return Builder.CreateCall(Fn, Ops, CI->getName());
} }
llvm_unreachable("Unknown function for ARM CallBase upgrade."); llvm_unreachable("Unknown function for ARM CallBase upgrade.");
} }
static Value *UpgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
Function *F, IRBuilder<> &Builder) {
const bool IsInc = Name.startswith("atomic.inc.");
if (IsInc || Name.startswith("atomic.dec.")) {
if (CI->getNumOperands() != 6) // Malformed bitcode.
return nullptr;
AtomicRMWInst::BinOp RMWOp =
IsInc ? AtomicRMWInst::UIncWrap : AtomicRMWInst::UDecWrap;
Value *Ptr = CI->getArgOperand(0);
Value *Val = CI->getArgOperand(1);
ConstantInt *OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
if (Order == AtomicOrdering::NotAtomic ||
Order == AtomicOrdering::Unordered)
Order = AtomicOrdering::SequentiallyConsistent;
AtomicRMWInst *RMW = Builder.CreateAtomicRMW(RMWOp, Ptr, Val, None, Order);
if (!VolatileArg || !VolatileArg->isZero())
RMW->setVolatile(true);
return RMW;
}
llvm_unreachable("Unknown function for AMDGPU intrinsic upgrade.");
}
/// Upgrade a call to an old intrinsic. All argument and return casting must be /// Upgrade a call to an old intrinsic. All argument and return casting must be
/// provided to seamlessly integrate with existing context. /// provided to seamlessly integrate with existing context.
void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
// Note dyn_cast to Function is not quite the same as getCalledFunction, which // Note dyn_cast to Function is not quite the same as getCalledFunction, which
// checks the callee's function type matches. It's likely we need to handle // checks the callee's function type matches. It's likely we need to handle
// type changes here. // type changes here.
Function *F = dyn_cast<Function>(CI->getCalledOperand()); Function *F = dyn_cast<Function>(CI->getCalledOperand());
if (!F) if (!F)
Show All 14 Lines if (!NewFn) {
if (IsX86) if (IsX86)
Name = Name.substr(4); Name = Name.substr(4);
bool IsNVVM = Name.startswith("nvvm."); bool IsNVVM = Name.startswith("nvvm.");
if (IsNVVM) if (IsNVVM)
Name = Name.substr(5); Name = Name.substr(5);
bool IsARM = Name.startswith("arm."); bool IsARM = Name.startswith("arm.");
if (IsARM) if (IsARM)
Name = Name.substr(4); Name = Name.substr(4);
bool IsAMDGCN = Name.startswith("amdgcn.");
if (IsAMDGCN)
Name = Name.substr(7);
if (IsX86 && Name.startswith("sse4a.movnt.")) { if (IsX86 && Name.startswith("sse4a.movnt.")) {
Module *M = F->getParent(); Module *M = F->getParent();
SmallVector<Metadata *, 1> Elts; SmallVector<Metadata *, 1> Elts;
Elts.push_back( Elts.push_back(
ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
MDNode *Node = MDNode::get(C, Elts); MDNode *Node = MDNode::get(C, Elts);
▲ Show 20 Lines • Show All 1,806 Lines • ▼ Show 20 Lines
Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc"); Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
} else if (IsNVVM && Name == "h2f") { } else if (IsNVVM && Name == "h2f") {
Rep = Builder.CreateCall(Intrinsic::getDeclaration( Rep = Builder.CreateCall(Intrinsic::getDeclaration(
F->getParent(), Intrinsic::convert_from_fp16, F->getParent(), Intrinsic::convert_from_fp16,
{Builder.getFloatTy()}), {Builder.getFloatTy()}),
CI->getArgOperand(0), "h2f"); CI->getArgOperand(0), "h2f");
} else if (IsARM) { } else if (IsARM) {
Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder); Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder);
} else if (IsAMDGCN) {
Rep = UpgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
} else { } else {
llvm_unreachable("Unknown function for CallBase upgrade."); llvm_unreachable("Unknown function for CallBase upgrade.");
} }
if (Rep) if (Rep)
CI->replaceAllUsesWith(Rep); CI->replaceAllUsesWith(Rep);
CI->eraseFromParent(); CI->eraseFromParent();
return; return;
▲ Show 20 Lines • Show All 1,046 Lines • Show Last 20 Lines

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h

Show First 20 Lines • Show All 192 Lines • ▼ Show 20 Linespublic:
bool legalizeImageIntrinsic( bool legalizeImageIntrinsic(
MachineInstr &MI, MachineIRBuilder &B, MachineInstr &MI, MachineIRBuilder &B,
GISelChangeObserver &Observer, GISelChangeObserver &Observer,
const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const; const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const;
bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const; bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const;
bool legalizeAtomicIncDec(MachineInstr &MI, MachineIRBuilder &B,
bool IsInc) const;
bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const; MachineIRBuilder &B) const;
bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const; MachineIRBuilder &B) const;
bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const; MachineIRBuilder &B) const;
bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const; MachineIRBuilder &B) const;
bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const; MachineIRBuilder &B) const;
bool legalizeIntrinsic(LegalizerHelper &Helper, bool legalizeIntrinsic(LegalizerHelper &Helper,
MachineInstr &MI) const override; MachineInstr &MI) const override;
}; };
} // End llvm namespace. } // End llvm namespace.
#endif #endif

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Show First 20 Lines • Show All 4,617 Lines • ▼ Show 20 Lines if (IsTFE) {
buildBufferLoad(Opc, Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format, buildBufferLoad(Opc, Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format,
AuxiliaryData, MMO, IsTyped, HasVIndex, B); AuxiliaryData, MMO, IsTyped, HasVIndex, B);
} }
MI.eraseFromParent(); MI.eraseFromParent();
return true; return true;
} }
bool AMDGPULegalizerInfo::legalizeAtomicIncDec(MachineInstr &MI,
MachineIRBuilder &B,
bool IsInc) const {
unsigned Opc = IsInc ? AMDGPU::G_ATOMICRMW_UINC_WRAP :
AMDGPU::G_ATOMICRMW_UDEC_WRAP;
B.buildInstr(Opc)
.addDef(MI.getOperand(0).getReg())
.addUse(MI.getOperand(2).getReg())
.addUse(MI.getOperand(3).getReg())
.cloneMemRefs(MI);
MI.eraseFromParent();
return true;
}
static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) { static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
switch (IntrID) { switch (IntrID) {
case Intrinsic::amdgcn_raw_buffer_atomic_swap: case Intrinsic::amdgcn_raw_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_buffer_atomic_swap: case Intrinsic::amdgcn_struct_buffer_atomic_swap:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP; return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
case Intrinsic::amdgcn_raw_buffer_atomic_add: case Intrinsic::amdgcn_raw_buffer_atomic_add:
case Intrinsic::amdgcn_struct_buffer_atomic_add: case Intrinsic::amdgcn_struct_buffer_atomic_add:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD; return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
▲ Show 20 Lines • Show All 1,127 Lines • ▼ Show 20 Linesbool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap: case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_buffer_atomic_fmin: case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
case Intrinsic::amdgcn_struct_buffer_atomic_fmin: case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
case Intrinsic::amdgcn_raw_buffer_atomic_fmax: case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
case Intrinsic::amdgcn_struct_buffer_atomic_fmax: case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
case Intrinsic::amdgcn_raw_buffer_atomic_fadd: case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_buffer_atomic_fadd: case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
return legalizeBufferAtomic(MI, B, IntrID); return legalizeBufferAtomic(MI, B, IntrID);
case Intrinsic::amdgcn_atomic_inc:
return legalizeAtomicIncDec(MI, B, true);
case Intrinsic::amdgcn_atomic_dec:
return legalizeAtomicIncDec(MI, B, false);
case Intrinsic::trap: case Intrinsic::trap:
return legalizeTrapIntrinsic(MI, MRI, B); return legalizeTrapIntrinsic(MI, MRI, B);
case Intrinsic::debugtrap: case Intrinsic::debugtrap:
return legalizeDebugTrapIntrinsic(MI, MRI, B); return legalizeDebugTrapIntrinsic(MI, MRI, B);
case Intrinsic::amdgcn_rsq_clamp: case Intrinsic::amdgcn_rsq_clamp:
return legalizeRsqClampIntrinsic(MI, MRI, B); return legalizeRsqClampIntrinsic(MI, MRI, B);
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
Show All 14 Lines

llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td

Show First 20 Lines • Show All 231 Lines • ▼ Show 20 Lines
def : SourceOfDivergence<int_amdgcn_interp_p2_f16>; def : SourceOfDivergence<int_amdgcn_interp_p2_f16>;
def : SourceOfDivergence<int_amdgcn_lds_direct_load>; def : SourceOfDivergence<int_amdgcn_lds_direct_load>;
def : SourceOfDivergence<int_amdgcn_lds_param_load>; def : SourceOfDivergence<int_amdgcn_lds_param_load>;
def : SourceOfDivergence<int_amdgcn_mbcnt_hi>; def : SourceOfDivergence<int_amdgcn_mbcnt_hi>;
def : SourceOfDivergence<int_amdgcn_mbcnt_lo>; def : SourceOfDivergence<int_amdgcn_mbcnt_lo>;
def : SourceOfDivergence<int_r600_read_tidig_x>; def : SourceOfDivergence<int_r600_read_tidig_x>;
def : SourceOfDivergence<int_r600_read_tidig_y>; def : SourceOfDivergence<int_r600_read_tidig_y>;
def : SourceOfDivergence<int_r600_read_tidig_z>; def : SourceOfDivergence<int_r600_read_tidig_z>;
def : SourceOfDivergence<int_amdgcn_atomic_inc>;
def : SourceOfDivergence<int_amdgcn_atomic_dec>;
def : SourceOfDivergence<int_amdgcn_global_atomic_csub>; def : SourceOfDivergence<int_amdgcn_global_atomic_csub>;
def : SourceOfDivergence<int_amdgcn_global_atomic_fadd>; def : SourceOfDivergence<int_amdgcn_global_atomic_fadd>;
def : SourceOfDivergence<int_amdgcn_global_atomic_fmin>; def : SourceOfDivergence<int_amdgcn_global_atomic_fmin>;
def : SourceOfDivergence<int_amdgcn_global_atomic_fmax>; def : SourceOfDivergence<int_amdgcn_global_atomic_fmax>;
def : SourceOfDivergence<int_amdgcn_flat_atomic_fadd>; def : SourceOfDivergence<int_amdgcn_flat_atomic_fadd>;
def : SourceOfDivergence<int_amdgcn_flat_atomic_fmin>; def : SourceOfDivergence<int_amdgcn_flat_atomic_fmin>;
def : SourceOfDivergence<int_amdgcn_flat_atomic_fmax>; def : SourceOfDivergence<int_amdgcn_flat_atomic_fmax>;
def : SourceOfDivergence<int_amdgcn_global_atomic_fadd_v2bf16>; def : SourceOfDivergence<int_amdgcn_global_atomic_fadd_v2bf16>;
▲ Show 20 Lines • Show All 126 Lines • Show Last 20 Lines

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

Show First 20 Lines • Show All 478 Lines • ▼ Show 20 Lines if (VF == 1)
return 1; return 1;
return 8; return 8;
} }
bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
MemIntrinsicInfo &Info) const { MemIntrinsicInfo &Info) const {
switch (Inst->getIntrinsicID()) { switch (Inst->getIntrinsicID()) {
case Intrinsic::amdgcn_atomic_inc:
case Intrinsic::amdgcn_atomic_dec:
case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: case Intrinsic::amdgcn_ds_ordered_swap:
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
case Intrinsic::amdgcn_ds_fmax: { case Intrinsic::amdgcn_ds_fmax: {
auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2)); auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2));
auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4)); auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4));
if (!Ordering || !Volatile) if (!Ordering || !Volatile)
▲ Show 20 Lines • Show All 507 Lines • ▼ Show 20 Lines if (CI->isInlineAsm())
return !isInlineAsmSourceOfDivergence(CI, ExtValue->getIndices()); return !isInlineAsmSourceOfDivergence(CI, ExtValue->getIndices());
return false; return false;
} }
bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
Intrinsic::ID IID) const { Intrinsic::ID IID) const {
switch (IID) { switch (IID) {
case Intrinsic::amdgcn_atomic_inc:
case Intrinsic::amdgcn_atomic_dec:
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
case Intrinsic::amdgcn_ds_fmax: case Intrinsic::amdgcn_ds_fmax:
case Intrinsic::amdgcn_is_shared: case Intrinsic::amdgcn_is_shared:
case Intrinsic::amdgcn_is_private: case Intrinsic::amdgcn_is_private:
case Intrinsic::amdgcn_flat_atomic_fadd: case Intrinsic::amdgcn_flat_atomic_fadd:
case Intrinsic::amdgcn_flat_atomic_fmax: case Intrinsic::amdgcn_flat_atomic_fmax:
case Intrinsic::amdgcn_flat_atomic_fmin: case Intrinsic::amdgcn_flat_atomic_fmin:
OpIndexes.push_back(0); OpIndexes.push_back(0);
return true; return true;
default: default:
return false; return false;
} }
} }
Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
Value *OldV, Value *OldV,
Value *NewV) const { Value *NewV) const {
auto IntrID = II->getIntrinsicID(); auto IntrID = II->getIntrinsicID();
switch (IntrID) { switch (IntrID) {
case Intrinsic::amdgcn_atomic_inc:
case Intrinsic::amdgcn_atomic_dec:
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
case Intrinsic::amdgcn_ds_fmax: { case Intrinsic::amdgcn_ds_fmax: {
const ConstantInt *IsVolatile = cast<ConstantInt>(II->getArgOperand(4)); const ConstantInt *IsVolatile = cast<ConstantInt>(II->getArgOperand(4));
if (!IsVolatile->isZero()) if (!IsVolatile->isZero())
return nullptr; return nullptr;
Module *M = II->getParent()->getParent()->getParent(); Module *M = II->getParent()->getParent()->getParent();
Type *DestTy = II->getType(); Type *DestTy = II->getType();
▲ Show 20 Lines • Show All 195 Lines • Show Last 20 Lines

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 1,057 Lines • ▼ Show 20 Lines if (ME.onlyReadsMemory()) {
return true; return true;
} }
} }
} }
return true; return true;
} }
switch (IntrID) { switch (IntrID) {
case Intrinsic::amdgcn_atomic_inc:
case Intrinsic::amdgcn_atomic_dec:
case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: case Intrinsic::amdgcn_ds_ordered_swap:
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
case Intrinsic::amdgcn_ds_fmax: { case Intrinsic::amdgcn_ds_fmax: {
Info.opc = ISD::INTRINSIC_W_CHAIN; Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.memVT = MVT::getVT(CI.getType()); Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0); Info.ptrVal = CI.getOperand(0);
▲ Show 20 Lines • Show All 125 Lines • ▼ Show 20 Lines default:
return false; return false;
} }
} }
bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II, bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
SmallVectorImpl<Value*> &Ops, SmallVectorImpl<Value*> &Ops,
Type *&AccessTy) const { Type *&AccessTy) const {
switch (II->getIntrinsicID()) { switch (II->getIntrinsicID()) {
case Intrinsic::amdgcn_atomic_inc:
case Intrinsic::amdgcn_atomic_dec:
case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: case Intrinsic::amdgcn_ds_ordered_swap:
case Intrinsic::amdgcn_ds_append: case Intrinsic::amdgcn_ds_append:
case Intrinsic::amdgcn_ds_consume: case Intrinsic::amdgcn_ds_consume:
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
case Intrinsic::amdgcn_ds_fmax: case Intrinsic::amdgcn_ds_fmax:
case Intrinsic::amdgcn_global_atomic_fadd: case Intrinsic::amdgcn_global_atomic_fadd:
▲ Show 20 Lines • Show All 6,097 Lines • ▼ Show 20 Lines SDValue Ops[] = {
Value, Value,
DAG.getTargetConstant(Offset, DL, MVT::i16), DAG.getTargetConstant(Offset, DL, MVT::i16),
copyToM0(DAG, Chain, DL, M0).getValue(1), // Glue copyToM0(DAG, Chain, DL, M0).getValue(1), // Glue
}; };
return DAG.getMemIntrinsicNode(AMDGPUISD::DS_ORDERED_COUNT, DL, return DAG.getMemIntrinsicNode(AMDGPUISD::DS_ORDERED_COUNT, DL,
M->getVTList(), Ops, M->getMemoryVT(), M->getVTList(), Ops, M->getMemoryVT(),
M->getMemOperand()); M->getMemOperand());
} }
case Intrinsic::amdgcn_atomic_inc:
case Intrinsic::amdgcn_atomic_dec:
case Intrinsic::amdgcn_ds_fadd: { case Intrinsic::amdgcn_ds_fadd: {
MemSDNode *M = cast<MemSDNode>(Op); MemSDNode *M = cast<MemSDNode>(Op);
unsigned Opc; unsigned Opc;
switch (IntrID) { switch (IntrID) {
case Intrinsic::amdgcn_ds_fadd: case Intrinsic::amdgcn_ds_fadd:
Opc = ISD::ATOMIC_LOAD_FADD; Opc = ISD::ATOMIC_LOAD_FADD;
break; break;
case Intrinsic::amdgcn_atomic_inc:
Opc = ISD::ATOMIC_LOAD_UINC_WRAP;
break;
case Intrinsic::amdgcn_atomic_dec:
Opc = ISD::ATOMIC_LOAD_UDEC_WRAP;
break;
} }
return DAG.getAtomic(Opc, SDLoc(Op), M->getMemoryVT(), return DAG.getAtomic(Opc, SDLoc(Op), M->getMemoryVT(),
M->getOperand(0), M->getOperand(2), M->getOperand(3), M->getOperand(0), M->getOperand(2), M->getOperand(3),
M->getMemOperand()); M->getMemOperand());
} }
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
case Intrinsic::amdgcn_ds_fmax: { case Intrinsic::amdgcn_ds_fmax: {
MemSDNode *M = cast<MemSDNode>(Op); MemSDNode *M = cast<MemSDNode>(Op);
unsigned Opc; unsigned Opc;
switch (IntrID) { switch (IntrID) {
case Intrinsic::amdgcn_atomic_inc:
Opc = ISD::ATOMIC_LOAD_UINC_WRAP;
break;
case Intrinsic::amdgcn_atomic_dec:
Opc = ISD::ATOMIC_LOAD_UDEC_WRAP;
break;
case Intrinsic::amdgcn_ds_fmin: case Intrinsic::amdgcn_ds_fmin:
Opc = AMDGPUISD::ATOMIC_LOAD_FMIN; Opc = AMDGPUISD::ATOMIC_LOAD_FMIN;
break; break;
case Intrinsic::amdgcn_ds_fmax: case Intrinsic::amdgcn_ds_fmax:
Opc = AMDGPUISD::ATOMIC_LOAD_FMAX; Opc = AMDGPUISD::ATOMIC_LOAD_FMAX;
break; break;
default: default:
llvm_unreachable("Unknown intrinsic!"); llvm_unreachable("Unknown intrinsic!");
▲ Show 20 Lines • Show All 5,966 Lines • Show Last 20 Lines

llvm/test/Analysis/DivergenceAnalysis/AMDGPU/atomics.ll

Show All 10 Lines
; CHECK: DIVERGENT: %orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst ; CHECK: DIVERGENT: %orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst
define amdgpu_kernel void @test2(ptr %ptr, i32 %cmp, i32 %new) { define amdgpu_kernel void @test2(ptr %ptr, i32 %cmp, i32 %new) {
%orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst %orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst
%val = extractvalue { i32, i1 } %orig, 0 %val = extractvalue { i32, i1 } %orig, 0
store i32 %val, ptr %ptr store i32 %val, ptr %ptr
ret void ret void
} }
; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
define i32 @test_atomic_inc_i32(ptr addrspace(1) %ptr, i32 %val) #0 {
%ret = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK: DIVERGENT: %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
define i64 @test_atomic_inc_i64(ptr addrspace(1) %ptr, i64 %val) #0 {
%ret = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
ret i64 %ret
}
; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
define i32 @test_atomic_dec_i32(ptr addrspace(1) %ptr, i32 %val) #0 {
%ret = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK: DIVERGENT: %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
define i64 @test_atomic_dec_i64(ptr addrspace(1) %ptr, i64 %val) #0 {
%ret = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
ret i64 %ret
}
declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #1
declare i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #1
declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #1
declare i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #1
; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val) ; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val)
define amdgpu_kernel void @test_atomic_csub_i32(ptr addrspace(1) %ptr, i32 %val) #0 { define amdgpu_kernel void @test_atomic_csub_i32(ptr addrspace(1) %ptr, i32 %val) #0 {
%ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val) %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val)
store i32 %ret, ptr addrspace(1) %ptr, align 4 store i32 %ret, i32 addrspace(1)* %ptr, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) nocapture, i32) #1 declare i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) nocapture, i32) #1
attributes #0 = { nounwind } attributes #0 = { nounwind }
attributes #1 = { argmemonly nounwind willreturn } attributes #1 = { argmemonly nounwind willreturn }

llvm/test/Analysis/LegacyDivergenceAnalysis/AMDGPU/atomics.ll

Show All 9 Lines
; CHECK: DIVERGENT: %orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst ; CHECK: DIVERGENT: %orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst
define amdgpu_kernel void @test2(ptr %ptr, i32 %cmp, i32 %new) { define amdgpu_kernel void @test2(ptr %ptr, i32 %cmp, i32 %new) {
%orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst %orig = cmpxchg ptr %ptr, i32 %cmp, i32 %new seq_cst seq_cst
%val = extractvalue { i32, i1 } %orig, 0 %val = extractvalue { i32, i1 } %orig, 0
store i32 %val, ptr %ptr store i32 %val, ptr %ptr
ret void ret void
} }
; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
define i32 @test_atomic_inc_i32(ptr addrspace(1) %ptr, i32 %val) #0 {
%ret = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK: DIVERGENT: %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
define i64 @test_atomic_inc_i64(ptr addrspace(1) %ptr, i64 %val) #0 {
%ret = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
ret i64 %ret
}
; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
define i32 @test_atomic_dec_i32(ptr addrspace(1) %ptr, i32 %val) #0 {
%ret = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 %val, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK: DIVERGENT: %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
define i64 @test_atomic_dec_i64(ptr addrspace(1) %ptr, i64 %val) #0 {
%ret = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 %val, i32 0, i32 0, i1 false)
ret i64 %ret
}
declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #1
declare i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #1
declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #1
declare i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #1
; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val) ; CHECK: DIVERGENT: %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val)
define amdgpu_kernel void @test_atomic_csub_i32(ptr addrspace(1) %ptr, i32 %val) #0 { define amdgpu_kernel void @test_atomic_csub_i32(i32 addrspace(1)* %ptr, i32 %val) #0 {
%ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val) %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %ptr, i32 %val)
store i32 %ret, ptr addrspace(1) %ptr, align 4 store i32 %ret, ptr addrspace(1) %ptr, align 4
ret void ret void
} }
declare i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) nocapture, i32) #1 declare i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) nocapture, i32) #1
attributes #0 = { nounwind } attributes #0 = { nounwind }
attributes #1 = { argmemonly nounwind willreturn } attributes #1 = { argmemonly nounwind willreturn }

llvm/test/Bitcode/amdgcn-atomic.ll

  • This file was added.
; RUN: llvm-as < %s | llvm-dis | FileCheck %s
define void @atomic_inc(ptr %ptr0, ptr addrspace(1) %ptr1, ptr addrspace(3) %ptr3) {
; CHECK: atomicrmw uinc_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr0, i32 42, i32 0, i32 0, i1 false)
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 0, i32 0, i1 false)
; CHECK: atomicrmw uinc_wrap ptr addrspace(3) %ptr3, i32 46 seq_cst, align 4
%result2 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr3, i32 46, i32 0, i32 0, i1 false)
; CHECK: atomicrmw uinc_wrap ptr %ptr0, i64 48 seq_cst, align 8
%result3 = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr0, i64 48, i32 0, i32 0, i1 false)
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i64 45 seq_cst, align 8
%result4 = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr1, i64 45, i32 0, i32 0, i1 false)
; CHECK: atomicrmw uinc_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
%result5 = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 false)
; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
%result6 = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 true)
ret void
}
define void @atomic_dec(ptr %ptr0, ptr addrspace(1) %ptr1, ptr addrspace(3) %ptr3) {
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 0, i1 false)
; CHECK: atomicrmw udec_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 0, i32 0, i1 false)
; CHECK: atomicrmw udec_wrap ptr addrspace(3) %ptr3, i32 46 seq_cst, align 4
%result2 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr3, i32 46, i32 0, i32 0, i1 false)
; CHECK: atomicrmw udec_wrap ptr %ptr0, i64 48 seq_cst, align 8
%result3 = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %ptr0, i64 48, i32 0, i32 0, i1 false)
; CHECK: atomicrmw udec_wrap ptr addrspace(1) %ptr1, i64 45 seq_cst, align 8
%result4 = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr1, i64 45, i32 0, i32 0, i1 false)
; CHECK: atomicrmw udec_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
%result5 = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 false)
; CHECK: atomicrmw volatile udec_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
%result6 = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 true)
ret void
}
; Test some invalid ordering handling
define void @ordering(ptr %ptr0, ptr addrspace(1) %ptr1, ptr addrspace(3) %ptr3) {
; CHECK: atomicrmw volatile uinc_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr0, i32 42, i32 -1, i32 0, i1 true)
; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 0, i32 0, i1 true)
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
%result2 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 1, i32 0, i1 false)
; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(1) %ptr1, i32 43 monotonic, align 4
%result3 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 2, i32 0, i1 true)
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
%result4 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 3, i32 0, i1 false)
; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result5 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 4, i1 true)
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result6 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 5, i1 false)
; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result7 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 6, i1 true)
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result8 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 7, i1 false)
; CHECK:= atomicrmw volatile udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result9 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 8, i1 true)
; CHECK:= atomicrmw volatile udec_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
%result10 = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 3, i32 0, i1 true)
ret void
}
define void @immarg_violations(ptr %ptr0, i32 %val32, i1 %val1) {
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 %val32, i32 0, i1 false)
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 monotonic, align 4
%result1 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 2, i32 %val32, i1 false)
; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 monotonic, align 4
%result2 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 2, i32 0, i1 %val1)
ret void
}
declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
attributes #0 = { argmemonly nounwind willreturn }

llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -mattr=+flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s
; FIXME: Merge with other test. DS offset folding doesn't work due to
; register bank copies, and no return optimization is missing.
@lds0 = internal addrspace(3) global [512 x i32] undef
@lds1 = internal addrspace(3) global [512 x i64] undef, align 8
declare i32 @llvm.amdgcn.workitem.id.x() #0
define amdgpu_kernel void @lds_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i32:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_rtn_u32 v2, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_ret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_rtn_u32 v2, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_rtn_u32 v0, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_ret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_rtn_u32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_ret_i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_rtn_u32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i32 addrspace(3)* %ptr, i32 42 seq_cst, align 4
store i32 %result, i32 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i32_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_rtn_u32 v2, v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_ret_i32_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_rtn_u32 v2, v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4
%result = atomicrmw udec_wrap i32 addrspace(3)* %gep, i32 42 seq_cst, align 4
store i32 %result, i32 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @lds_atomic_dec_noret_i32(i32 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_noret_i32:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_u32 v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_noret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_u32 v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_u32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_noret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_u32 v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_noret_i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_u32 v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i32 addrspace(3)* %ptr, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(i32 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_noret_i32_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_u32 v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_noret_i32_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_u32 v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_noret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_u32 v1, v0 offset:16
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_noret_i32_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_u32 v1, v0 offset:16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_noret_i32_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_u32 v1, v0 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4
%result = atomicrmw udec_wrap i32 addrspace(3)* %gep, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_ret_i32:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_ret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec v0, v1, v0, s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_ret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_ret_i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i32 addrspace(1)* %ptr, i32 42 seq_cst, align 4
store i32 %result, i32 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_ret_i32_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 16
; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_ret_i32_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 16
; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_ret_i32_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_ret_i32_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] offset:16 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
%result = atomicrmw udec_wrap i32 addrspace(1)* %gep, i32 42 seq_cst, align 4
store i32 %result, i32 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_noret_i32(i32 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_noret_i32:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_noret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec v1, v0, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_noret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_noret_i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i32 addrspace(1)* %ptr, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(i32 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_noret_i32_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 16
; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_noret_i32_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_noret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_noret_i32_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_noret_i32_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
%result = atomicrmw udec_wrap i32 addrspace(1)* %gep, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_ret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: v_mov_b32_e32 v3, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_ret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: v_mov_b32_e32 v3, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_ret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:20 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_ret_i32_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:20 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_ret_i32_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u32 v1, v0, v1, s[2:3] offset:20 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id
%out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %id
%gep = getelementptr i32, i32 addrspace(1)* %gep.tid, i32 5
%result = atomicrmw udec_wrap i32 addrspace(1)* %gep, i32 42 seq_cst, align 4
store i32 %result, i32 addrspace(1)* %out.gep, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(i32 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec v0, v1, s[0:1] offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec v0, v1, s[0:1] offset:20
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:20
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, i32 addrspace(1)* %ptr, i32 %id
%gep = getelementptr i32, i32 addrspace(1)* %gep.tid, i32 5
%result = atomicrmw udec_wrap i32 addrspace(1)* %gep, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_ret_i32(i32* %out, i32* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_ret_i32:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_ret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: flat_store_dword v[0:1], v2
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_ret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: flat_store_dword v[0:1], v2
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_ret_i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: flat_store_b32 v[0:1], v2
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i32* %ptr, i32 42 seq_cst, align 4
store i32 %result, i32* %out, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(i32* %out, i32* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_ret_i32_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 16
; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_ret_i32_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 16
; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: flat_store_dword v[0:1], v2
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_ret_i32_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s2, s2, 16
; GFX10-NEXT: s_addc_u32 s3, s3, 0
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: flat_store_dword v[0:1], v2
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_ret_i32_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: flat_store_b32 v[0:1], v2
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, i32* %ptr, i32 4
%result = atomicrmw udec_wrap i32* %gep, i32 42 seq_cst, align 4
store i32 %result, i32* %out, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_noret_i32(i32* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_noret_i32:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_noret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec v[0:1], v2
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_noret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_noret_i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i32* %ptr, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(i32* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_noret_i32_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 16
; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_noret_i32_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_noret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_noret_i32_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s0, s0, 16
; GFX10-NEXT: s_addc_u32 s1, s1, 0
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_noret_i32_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, i32* %ptr, i32 4
%result = atomicrmw udec_wrap i32* %gep, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(i32* %out, i32* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: v_mov_b32_e32 v3, 42
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: v_mov_b32_e32 v3, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec v3, v[0:1], v3 offset:20 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: flat_store_dword v[0:1], v3
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec v3, v[0:1], v3 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: flat_store_dword v[0:1], v3
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v3, 42 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u32 v3, v[0:1], v3 offset:20 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: flat_store_b32 v[0:1], v3
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, i32* %ptr, i32 %id
%out.gep = getelementptr i32, i32* %out, i32 %id
%gep = getelementptr i32, i32* %gep.tid, i32 5
%result = atomicrmw udec_wrap i32* %gep, i32 42 seq_cst, align 4
store i32 %result, i32* %out.gep, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(i32* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:20
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, i32* %ptr, i32 %id
%gep = getelementptr i32, i32* %gep.tid, i32 5
%result = atomicrmw udec_wrap i32* %gep, i32 42 seq_cst, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_ret_i64(i64* %out, i64* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_ret_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_ret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_ret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_ret_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2
; GFX11-NEXT: v_mov_b32_e32 v3, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i64* %ptr, i64 42 seq_cst, align 8
store i64 %result, i64* %out, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(i64* %out, i64* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_ret_i64_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 32
; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_ret_i64_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 32
; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_ret_i64_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s2, s2, 32
; GFX10-NEXT: s_addc_u32 s3, s3, 0
; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_ret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2
; GFX11-NEXT: v_mov_b32_e32 v3, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] offset:32 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, i64* %ptr, i32 4
%result = atomicrmw udec_wrap i64* %gep, i64 42 seq_cst, align 8
store i64 %result, i64* %out, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_noret_i64(i64* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_noret_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_noret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_noret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_noret_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i64* %ptr, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(i64* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_noret_i64_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 32
; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_noret_i64_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_noret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_noret_i64_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s0, s0, 32
; GFX10-NEXT: s_addc_u32 s1, s1, 0
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_noret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, i64* %ptr, i32 4
%result = atomicrmw udec_wrap i64* %gep, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(i64* %out, i64* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s3
; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[3:4], v[1:2] offset:40 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] offset:40 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, i64* %ptr, i32 %id
%out.gep = getelementptr i64, i64* %out, i32 %id
%gep = getelementptr i64, i64* %gep.tid, i32 5
%result = atomicrmw udec_wrap i64* %gep, i64 42 seq_cst, align 8
store i64 %result, i64* %out.gep, align 4
ret void
}
define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(i64* %ptr) #1 {
; CI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s1
; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v0
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_dec_x2 v[3:4], v[1:2] offset:40
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s0
; GFX11-NEXT: v_mov_b32_e32 v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] offset:40
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, i64* %ptr, i32 %id
%gep = getelementptr i64, i64* %gep.tid, i32 5
%result = atomicrmw udec_wrap i64* %gep, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #1 {
; CI-LABEL: atomic_dec_shl_base_lds_0:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; CI-NEXT: v_mov_b32_e32 v2, 9
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_rtn_u32 v2, v1, v2 offset:8
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_i32_e32 v3, vcc, 2, v0
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_shl_base_lds_0:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; VI-NEXT: v_mov_b32_e32 v2, 9
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_rtn_u32 v2, v1, v2 offset:8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_dec_shl_base_lds_0:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 9
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:8
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_store_dword v2, v0, s[2:3]
; GFX9-NEXT: global_store_dword v2, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: atomic_dec_shl_base_lds_0:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 9
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:8
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: global_store_dword v2, v0, s[2:3]
; GFX10-NEXT: global_store_dword v2, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_dec_shl_base_lds_0:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_dual_mov_b32 v2, 9 :: v_dual_lshlrev_b32 v1, 2, v0
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:8
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v2, v0, s[2:3]
; GFX11-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #2
%idx.0 = add nsw i32 %tid.x, 2
%arrayidx0 = getelementptr inbounds [512 x i32], [512 x i32] addrspace(3)* @lds0, i32 0, i32 %idx.0
%result = atomicrmw udec_wrap i32 addrspace(3)* %arrayidx0, i32 9 seq_cst, align 4
store i32 %idx.0, i32 addrspace(1)* %add_use, align 4
store i32 %result, i32 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @lds_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_ret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_ret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_ret_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i64 addrspace(3)* %ptr, i64 42 seq_cst, align 8
store i64 %result, i64 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i64_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_ret_i64_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, i64 addrspace(3)* %ptr, i32 4
%result = atomicrmw udec_wrap i64 addrspace(3)* %gep, i64 42 seq_cst, align 8
store i64 %result, i64 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @lds_atomic_dec_noret_i64(i64 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_noret_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_u64 v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_noret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_u64 v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_u64 v2, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_noret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_u64 v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_noret_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_u64 v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i64 addrspace(3)* %ptr, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(i64 addrspace(3)* %ptr) #1 {
; CI-LABEL: lds_atomic_dec_noret_i64_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_u64 v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_noret_i64_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_u64 v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_noret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_u64 v2, v[0:1] offset:32
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: lds_atomic_dec_noret_i64_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_u64 v2, v[0:1] offset:32
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_dec_noret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_u64 v2, v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, i64 addrspace(3)* %ptr, i32 4
%result = atomicrmw udec_wrap i64 addrspace(3)* %gep, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @global_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_ret_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_ret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_ret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_ret_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i64 addrspace(1)* %ptr, i64 42 seq_cst, align 8
store i64 %result, i64 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_ret_i64_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 32
; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_ret_i64_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 32
; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_ret_i64_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_ret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
%result = atomicrmw udec_wrap i64 addrspace(1)* %gep, i64 42 seq_cst, align 8
store i64 %result, i64 addrspace(1)* %out, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_noret_i64(i64 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_noret_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_noret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_noret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_noret_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%result = atomicrmw udec_wrap i64 addrspace(1)* %ptr, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(i64 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_noret_i64_offset:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 32
; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_noret_i64_offset:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_noret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:32
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_noret_i64_offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:32
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_noret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:32
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
%result = atomicrmw udec_wrap i64 addrspace(1)* %gep, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_ret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_ret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_ret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_ret_i64_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_ret_i64_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v1, 42
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 3, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v3, v[1:2], s[2:3] offset:40 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, i64 addrspace(1)* %ptr, i32 %id
%out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %id
%gep = getelementptr i64, i64 addrspace(1)* %gep.tid, i32 5
%result = atomicrmw udec_wrap i64 addrspace(1)* %gep, i64 42 seq_cst, align 8
store i64 %result, i64 addrspace(1)* %out.gep, align 4
ret void
}
define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(i64 addrspace(1)* %ptr) #1 {
; CI-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_dec_x2 v0, v[1:2], s[0:1] offset:40
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_dec_x2 v0, v[1:2], s[0:1] offset:40
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 3, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_dec_u64 v0, v[1:2], s[0:1] offset:40
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, i64 addrspace(1)* %ptr, i32 %id
%gep = getelementptr i64, i64 addrspace(1)* %gep.tid, i32 5
%result = atomicrmw udec_wrap i64 addrspace(1)* %gep, i64 42 seq_cst, align 8
ret void
}
define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #1 {
; CI-LABEL: atomic_dec_shl_base_lds_0_i64:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v1, 9
; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; CI-NEXT: v_mov_b32_e32 v2, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v4, s3
; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v0
; CI-NEXT: v_mov_b32_e32 v3, s2
; CI-NEXT: flat_store_dword v[3:4], v0
; CI-NEXT: v_mov_b32_e32 v4, s1
; CI-NEXT: v_mov_b32_e32 v3, s0
; CI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]
; CI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_shl_base_lds_0_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v1, 9
; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; VI-NEXT: v_mov_b32_e32 v2, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, s3
; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v0
; VI-NEXT: v_mov_b32_e32 v3, s2
; VI-NEXT: flat_store_dword v[3:4], v0
; VI-NEXT: v_mov_b32_e32 v4, s1
; VI-NEXT: v_mov_b32_e32 v3, s0
; VI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_dec_shl_base_lds_0_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, 9
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: global_store_dword v3, v0, s[2:3]
; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: atomic_dec_shl_base_lds_0_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v1, 9
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: global_store_dword v3, v0, s[2:3]
; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_dec_shl_base_lds_0_i64:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_mov_b32_e32 v1, 9
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 3, v0
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v3, v0, s[2:3]
; GFX11-NEXT: global_store_b64 v3, v[1:2], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #2
%idx.0 = add nsw i32 %tid.x, 2
%arrayidx0 = getelementptr inbounds [512 x i64], [512 x i64] addrspace(3)* @lds1, i32 0, i32 %idx.0
%result = atomicrmw udec_wrap i64 addrspace(3)* %arrayidx0, i64 9 seq_cst, align 8
store i32 %idx.0, i32 addrspace(1)* %add_use, align 4
store i64 %result, i64 addrspace(1)* %out, align 4
ret void
}
attributes #0 = { nounwind speculatable willreturn memory(none) }
attributes #1 = { nounwind }
attributes #2 = { nounwind memory(none) }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}

llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll

  • This file was moved from llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll.
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -mattr=+flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -mattr=+flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s
; FIXME: Merge with other test. DS offset folding doesn't work due to ; FIXME: Merge with other test. DS offset folding doesn't work due to
; register bank copies, and no return optimization is missing. ; register bank copies, and no return optimization is missing.
@lds0 = internal addrspace(3) global [512 x i32] undef, align 4
@lds1 = internal addrspace(3) global [512 x i64] undef, align 8
declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #2 declare i32 @llvm.amdgcn.workitem.id.x() #0
declare i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1) #2
declare i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr nocapture, i32, i32, i32, i1) #2
declare i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #2
declare i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) nocapture, i64, i32, i32, i1) #2
declare i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr nocapture, i64, i32, i32, i1) #2
declare i32 @llvm.amdgcn.workitem.id.x() #1
define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 { define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i32: ; CI-LABEL: lds_atomic_inc_ret_i32:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2 ; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2 ; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0 ; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_ret_i32: ; VI-LABEL: lds_atomic_inc_ret_i32:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8 ; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2 ; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0 ; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_ret_i32: ; GFX9-LABEL: lds_atomic_inc_ret_i32:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u32 v0, v0, v1 ; GFX9-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_store_dword v1, v0, s[0:1] ; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_ret_i32: ; GFX10-LABEL: lds_atomic_inc_ret_i32:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
cdevadasUnsubmitted
Not Done
ReplyInline Actions

There are additional waitcnt instructions in most of the lit pattern changes. Were they regressions earlier?

cdevadas: There are additional waitcnt instructions in most of the lit pattern changes. Were they…
arsenmAuthorUnsubmitted
Done
ReplyInline Actions

Without an ordering passed through to the memory legalizer, these weren’t actually treated as atomics. They were not properly legalized before

arsenm: Without an ordering passed through to the memory legalizer, these weren’t actually treated as…
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v1 ; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_ret_i32: ; GFX11-LABEL: lds_atomic_inc_ret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8 ; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2 ; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u32 v0, v0, v1 ; GFX11-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false), !noalias !0 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 seq_cst, align 4
store i32 %result, ptr addrspace(1) %out store i32 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
!0 = !{!1} define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
!1 = distinct !{!1, !2}
!2 = distinct !{!2}
define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_inc_ret_i32_offset: ; CI-LABEL: lds_atomic_inc_ret_i32_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2 ; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2 ; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16 ; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_ret_i32_offset: ; VI-LABEL: lds_atomic_inc_ret_i32_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8 ; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2 ; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16 ; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_ret_i32_offset: ; GFX9-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2 ; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 ; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_store_dword v1, v0, s[0:1] ; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_ret_i32_offset: ; GFX10-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 ; GFX10-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_ret_i32_offset: ; GFX11-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8 ; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s2 ; GFX11-NEXT: v_mov_b32_e32 v1, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16 ; GFX11-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i32 42 seq_cst, align 4
store i32 %result, ptr addrspace(1) %out store i32 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @lds_atomic_inc_noret_i32(ptr addrspace(3) %ptr) nounwind { define amdgpu_kernel void @lds_atomic_inc_noret_i32(ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_noret_i32: ; CI-LABEL: lds_atomic_inc_noret_i32:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0 ; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s0 ; CI-NEXT: v_mov_b32_e32 v1, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_u32 v1, v0 ; CI-NEXT: ds_inc_u32 v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_noret_i32: ; VI-LABEL: lds_atomic_inc_noret_i32:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0 ; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s0 ; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_u32 v1, v0 ; VI-NEXT: ds_inc_u32 v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_noret_i32: ; GFX9-LABEL: lds_atomic_inc_noret_i32:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_u32 v0, v1 ; GFX9-NEXT: ds_inc_u32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_noret_i32: ; GFX10-LABEL: lds_atomic_inc_noret_i32:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_u32 v0, v1 ; GFX10-NEXT: ds_inc_u32 v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_noret_i32: ; GFX11-LABEL: lds_atomic_inc_noret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 ; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0 ; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_u32 v0, v1 ; GFX11-NEXT: ds_inc_u32 v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 seq_cst, align 4
ret void ret void
} }
define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(ptr addrspace(3) %ptr) nounwind { define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_noret_i32_offset: ; CI-LABEL: lds_atomic_inc_noret_i32_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0 ; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s0 ; CI-NEXT: v_mov_b32_e32 v1, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_u32 v1, v0 offset:16 ; CI-NEXT: ds_inc_u32 v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_noret_i32_offset: ; VI-LABEL: lds_atomic_inc_noret_i32_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0 ; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s0 ; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_u32 v1, v0 offset:16 ; VI-NEXT: ds_inc_u32 v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_noret_i32_offset: ; GFX9-LABEL: lds_atomic_inc_noret_i32_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s0 ; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_u32 v1, v0 offset:16 ; GFX9-NEXT: ds_inc_u32 v1, v0 offset:16
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_noret_i32_offset: ; GFX10-LABEL: lds_atomic_inc_noret_i32_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_u32 v1, v0 offset:16 ; GFX10-NEXT: ds_inc_u32 v1, v0 offset:16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_noret_i32_offset: ; GFX11-LABEL: lds_atomic_inc_noret_i32_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 ; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0 ; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_u32 v1, v0 offset:16 ; GFX11-NEXT: ds_inc_u32 v1, v0 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i32 42 seq_cst, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_ret_i32: ; CI-LABEL: global_atomic_inc_ret_i32:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_ret_i32: ; VI-LABEL: global_atomic_inc_ret_i32:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_ret_i32: ; GFX9-LABEL: global_atomic_inc_ret_i32:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc ; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dword v1, v0, s[0:1] ; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_ret_i32: ; GFX10-LABEL: global_atomic_inc_ret_i32:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc ; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_ret_i32: ; GFX11-LABEL: global_atomic_inc_ret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0 ; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] glc ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 seq_cst, align 4
store i32 %result, ptr addrspace(1) %out store i32 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_ret_i32_offset: ; CI-LABEL: global_atomic_inc_ret_i32_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 16 ; CI-NEXT: s_add_u32 s2, s2, 16
; CI-NEXT: s_addc_u32 s3, s3, 0 ; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_ret_i32_offset: ; VI-LABEL: global_atomic_inc_ret_i32_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 16 ; VI-NEXT: s_add_u32 s2, s2, 16
; VI-NEXT: s_addc_u32 s3, s3, 0 ; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_ret_i32_offset: ; GFX9-LABEL: global_atomic_inc_ret_i32_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc ; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dword v1, v0, s[0:1] ; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_ret_i32_offset: ; GFX10-LABEL: global_atomic_inc_ret_i32_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc ; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_ret_i32_offset: ; GFX11-LABEL: global_atomic_inc_ret_i32_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0 ; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 glc ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
store i32 %result, ptr addrspace(1) %out store i32 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) nounwind { define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_noret_i32: ; CI-LABEL: global_atomic_inc_noret_i32:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v[0:1], v2 ; CI-NEXT: flat_atomic_inc v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_noret_i32: ; VI-LABEL: global_atomic_inc_noret_i32:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v[0:1], v2 ; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_noret_i32: ; GFX9-LABEL: global_atomic_inc_noret_i32:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1] ; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_noret_i32: ; GFX10-LABEL: global_atomic_inc_noret_i32:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] ; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_noret_i32: ; GFX11-LABEL: global_atomic_inc_noret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0 ; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 seq_cst, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %ptr) nounwind { define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_noret_i32_offset: ; CI-LABEL: global_atomic_inc_noret_i32_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 16 ; CI-NEXT: s_add_u32 s0, s0, 16
; CI-NEXT: s_addc_u32 s1, s1, 0 ; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v[0:1], v2 ; CI-NEXT: flat_atomic_inc v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_noret_i32_offset: ; VI-LABEL: global_atomic_inc_noret_i32_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 16 ; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0 ; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v[0:1], v2 ; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_noret_i32_offset: ; GFX9-LABEL: global_atomic_inc_noret_i32_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16 ; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_noret_i32_offset: ; GFX10-LABEL: global_atomic_inc_noret_i32_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16 ; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_noret_i32_offset: ; GFX11-LABEL: global_atomic_inc_noret_i32_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0 ; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16 ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_ret_i32_offset_addr64: ; CI-LABEL: global_atomic_inc_ret_i32_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: v_mov_b32_e32 v3, 42 ; CI-NEXT: v_mov_b32_e32 v3, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0 ; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc ; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v3 ; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_ret_i32_offset_addr64: ; VI-LABEL: global_atomic_inc_ret_i32_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: v_mov_b32_e32 v3, 42 ; VI-NEXT: v_mov_b32_e32 v3, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0 ; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc ; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v3 ; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_ret_i32_offset_addr64: ; GFX9-LABEL: global_atomic_inc_ret_i32_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc ; GFX9-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_ret_i32_offset_addr64: ; GFX10-LABEL: global_atomic_inc_ret_i32_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc ; GFX10-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_ret_i32_offset_addr64: ; GFX11-LABEL: global_atomic_inc_ret_i32_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 2, v0 ; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u32 v1, v0, v1, s[2:3] offset:20 glc ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, v1, s[2:3] offset:20 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
%out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id
%gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
store i32 %result, ptr addrspace(1) %out.gep store i32 %result, ptr addrspace(1) %out.gep, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_noret_i32_offset_addr64: ; CI-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0 ; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v[0:1], v2 ; CI-NEXT: flat_atomic_inc v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_noret_i32_offset_addr64: ; VI-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0 ; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v[0:1], v2 ; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_noret_i32_offset_addr64: ; GFX9-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1] offset:20 ; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1] offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_noret_i32_offset_addr64: ; GFX10-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc v0, v1, s[0:1] offset:20 ; GFX10-NEXT: global_atomic_inc v0, v1, s[0:1] offset:20
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_noret_i32_offset_addr64: ; GFX11-LABEL: global_atomic_inc_noret_i32_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 2, v0 ; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:20 ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:20
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
%gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
ret void ret void
} }
@lds0 = internal addrspace(3) global [512 x i32] undef, align 4 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #1 {
define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {
; CI-LABEL: atomic_inc_shl_base_lds_0_i32: ; CI-LABEL: atomic_inc_shl_base_lds_0_i32:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v0 ; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; CI-NEXT: v_mov_b32_e32 v2, 9 ; CI-NEXT: v_mov_b32_e32 v2, 9
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u32 v2, v1, v2 offset:8 ; CI-NEXT: ds_inc_rtn_u32 v2, v1, v2 offset:8
; CI-NEXT: v_add_i32_e32 v3, vcc, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_i32_e32 v3, vcc, 2, v0
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: flat_store_dword v[0:1], v3 ; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: atomic_inc_shl_base_lds_0_i32: ; VI-LABEL: atomic_inc_shl_base_lds_0_i32:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v0 ; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; VI-NEXT: v_mov_b32_e32 v2, 9 ; VI-NEXT: v_mov_b32_e32 v2, 9
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u32 v2, v1, v2 offset:8 ; VI-NEXT: ds_inc_rtn_u32 v2, v1, v2 offset:8
; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v0
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: flat_store_dword v[0:1], v3 ; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32: ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 9 ; GFX9-NEXT: v_mov_b32_e32 v2, 9
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8 ; GFX9-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, 2, v0 ; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v2, v0, s[2:3] ; GFX9-NEXT: global_store_dword v2, v0, s[2:3]
; GFX9-NEXT: global_store_dword v2, v1, s[0:1] ; GFX9-NEXT: global_store_dword v2, v1, s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: atomic_inc_shl_base_lds_0_i32: ; GFX10-LABEL: atomic_inc_shl_base_lds_0_i32:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 9 ; GFX10-NEXT: v_mov_b32_e32 v2, 9
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8 ; GFX10-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: global_store_dword v2, v0, s[2:3] ; GFX10-NEXT: global_store_dword v2, v0, s[2:3]
; GFX10-NEXT: global_store_dword v2, v1, s[0:1] ; GFX10-NEXT: global_store_dword v2, v1, s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: atomic_inc_shl_base_lds_0_i32: ; GFX11-LABEL: atomic_inc_shl_base_lds_0_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: v_dual_mov_b32 v2, 9 :: v_dual_lshlrev_b32 v1, 2, v0 ; GFX11-NEXT: v_dual_mov_b32 v2, 9 :: v_dual_lshlrev_b32 v1, 2, v0
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0 ; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8 ; GFX11-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v2, v0, s[2:3] ; GFX11-NEXT: global_store_b32 v2, v0, s[2:3]
; GFX11-NEXT: global_store_b32 v2, v1, s[0:1] ; GFX11-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #2
%idx.0 = add nsw i32 %tid.x, 2 %idx.0 = add nsw i32 %tid.x, 2
%arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.0 %arrayidx0 = getelementptr inbounds [512 x i32], pr addrspace(3) @lds0, i32 0, i32 %idx.0
%val0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %arrayidx0, i32 9, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %arrayidx0, i32 9 seq_cst, align 4
store i32 %idx.0, ptr addrspace(1) %add_use store i32 %idx.0, ptr addrspace(1) %add_use, align 4
store i32 %val0, ptr addrspace(1) %out store i32 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 { define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i64: ; CI-LABEL: lds_atomic_inc_ret_i64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2 ; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] ; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_ret_i64: ; VI-LABEL: lds_atomic_inc_ret_i64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8 ; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] ; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_ret_i64: ; GFX9-LABEL: lds_atomic_inc_ret_i64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] ; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_ret_i64: ; GFX10-LABEL: lds_atomic_inc_ret_i64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] ; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_ret_i64: ; GFX11-LABEL: lds_atomic_inc_ret_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8 ; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] ; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i64 42 seq_cst, align 8
store i64 %result, ptr addrspace(1) %out store i64 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 { define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i64_offset: ; CI-LABEL: lds_atomic_inc_ret_i64_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s2, s[4:5], 0x2 ; CI-NEXT: s_load_dword s2, s[4:5], 0x2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 ; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_ret_i64_offset: ; VI-LABEL: lds_atomic_inc_ret_i64_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s2, s[4:5], 0x8 ; VI-NEXT: s_load_dword s2, s[4:5], 0x8
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 ; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_ret_i64_offset: ; GFX9-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8 ; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 ; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_ret_i64_offset: ; GFX10-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 ; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_ret_i64_offset: ; GFX11-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8 ; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x8
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32 ; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i64 42 seq_cst, align 8
store i64 %result, ptr addrspace(1) %out store i64 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @lds_atomic_inc_noret_i64(ptr addrspace(3) %ptr) nounwind { define amdgpu_kernel void @lds_atomic_inc_noret_i64(ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_noret_i64: ; CI-LABEL: lds_atomic_inc_noret_i64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0 ; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_u64 v2, v[0:1] ; CI-NEXT: ds_inc_u64 v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_noret_i64: ; VI-LABEL: lds_atomic_inc_noret_i64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0 ; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_u64 v2, v[0:1] ; VI-NEXT: ds_inc_u64 v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_noret_i64: ; GFX9-LABEL: lds_atomic_inc_noret_i64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_u64 v2, v[0:1] ; GFX9-NEXT: ds_inc_u64 v2, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_noret_i64: ; GFX10-LABEL: lds_atomic_inc_noret_i64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_u64 v2, v[0:1] ; GFX10-NEXT: ds_inc_u64 v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_noret_i64: ; GFX11-LABEL: lds_atomic_inc_noret_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 ; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_u64 v2, v[0:1] ; GFX11-NEXT: ds_inc_u64 v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @lds_atomic_inc_noret_i64_offset(ptr addrspace(3) %ptr) nounwind { define amdgpu_kernel void @lds_atomic_inc_noret_i64_offset(ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_noret_i64_offset: ; CI-LABEL: lds_atomic_inc_noret_i64_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s0, s[4:5], 0x0 ; CI-NEXT: s_load_dword s0, s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; CI-NEXT: ds_inc_u64 v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: lds_atomic_inc_noret_i64_offset: ; VI-LABEL: lds_atomic_inc_noret_i64_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s0, s[4:5], 0x0 ; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; VI-NEXT: ds_inc_u64 v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: lds_atomic_inc_noret_i64_offset: ; GFX9-LABEL: lds_atomic_inc_noret_i64_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; GFX9-NEXT: ds_inc_u64 v2, v[0:1] offset:32
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: lds_atomic_inc_noret_i64_offset: ; GFX10-LABEL: lds_atomic_inc_noret_i64_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; GFX10-NEXT: ds_inc_u64 v2, v[0:1] offset:32
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: lds_atomic_inc_noret_i64_offset: ; GFX11-LABEL: lds_atomic_inc_noret_i64_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 ; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_u64 v2, v[0:1] offset:32 ; GFX11-NEXT: ds_inc_u64 v2, v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_ret_i64: ; CI-LABEL: global_atomic_inc_ret_i64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3 ; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_ret_i64: ; VI-LABEL: global_atomic_inc_ret_i64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3 ; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_ret_i64: ; GFX9-LABEL: global_atomic_inc_ret_i64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc ; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_ret_i64: ; GFX10-LABEL: global_atomic_inc_ret_i64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_ret_i64: ; GFX11-LABEL: global_atomic_inc_ret_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] glc ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 seq_cst, align 8
store i64 %result, ptr addrspace(1) %out store i64 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_ret_i64_offset: ; CI-LABEL: global_atomic_inc_ret_i64_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 32 ; CI-NEXT: s_add_u32 s2, s2, 32
; CI-NEXT: s_addc_u32 s3, s3, 0 ; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3 ; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_ret_i64_offset: ; VI-LABEL: global_atomic_inc_ret_i64_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 32 ; VI-NEXT: s_add_u32 s2, s2, 32
; VI-NEXT: s_addc_u32 s3, s3, 0 ; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3 ; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_ret_i64_offset: ; GFX9-LABEL: global_atomic_inc_ret_i64_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_ret_i64_offset: ; GFX10-LABEL: global_atomic_inc_ret_i64_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_ret_i64_offset: ; GFX11-LABEL: global_atomic_inc_ret_i64_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
store i64 %result, ptr addrspace(1) %out store i64 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) nounwind { define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_noret_i64: ; CI-LABEL: global_atomic_inc_noret_i64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_noret_i64: ; VI-LABEL: global_atomic_inc_noret_i64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_noret_i64: ; GFX9-LABEL: global_atomic_inc_noret_i64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] ; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_noret_i64: ; GFX10-LABEL: global_atomic_inc_noret_i64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] ; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_noret_i64: ; GFX11-LABEL: global_atomic_inc_noret_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] ; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %ptr) nounwind { define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_noret_i64_offset: ; CI-LABEL: global_atomic_inc_noret_i64_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 32 ; CI-NEXT: s_add_u32 s0, s0, 32
; CI-NEXT: s_addc_u32 s1, s1, 0 ; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_noret_i64_offset: ; VI-LABEL: global_atomic_inc_noret_i64_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 32 ; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0 ; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_noret_i64_offset: ; GFX9-LABEL: global_atomic_inc_noret_i64_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32 ; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_noret_i64_offset: ; GFX10-LABEL: global_atomic_inc_noret_i64_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32 ; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_noret_i64_offset: ; GFX11-LABEL: global_atomic_inc_noret_i64_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 ; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_ret_i64_offset_addr64: ; CI-LABEL: global_atomic_inc_ret_i64_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v4 ; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2 ; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_ret_i64_offset_addr64: ; VI-LABEL: global_atomic_inc_ret_i64_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v4 ; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2 ; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_ret_i64_offset_addr64: ; GFX9-LABEL: global_atomic_inc_ret_i64_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1] ; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_ret_i64_offset_addr64: ; GFX10-LABEL: global_atomic_inc_ret_i64_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1] ; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_ret_i64_offset_addr64: ; GFX11-LABEL: global_atomic_inc_ret_i64_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v1, 42 ; GFX11-NEXT: v_mov_b32_e32 v1, 42
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 3, v0 ; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 3, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v3, v[1:2], s[2:3] offset:40 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1] ; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
%out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id
%gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
store i64 %result, ptr addrspace(1) %out.gep store i64 %result, ptr addrspace(1) %out.gep, align 4
ret void ret void
} }
define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #0 { define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #1 {
; CI-LABEL: global_atomic_inc_noret_i64_offset_addr64: ; CI-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0 ; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2 ; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: global_atomic_inc_noret_i64_offset_addr64: ; VI-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2 ; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: global_atomic_inc_noret_i64_offset_addr64: ; GFX9-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:40 ; GFX9-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:40
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: global_atomic_inc_noret_i64_offset_addr64: ; GFX10-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:40 ; GFX10-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:40
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_atomic_inc_noret_i64_offset_addr64: ; GFX11-LABEL: global_atomic_inc_noret_i64_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 3, v0 ; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_lshlrev_b32 v0, 3, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 0 ; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[0:1] offset:40 ; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[0:1] offset:40
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
%gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {
; GCN-LABEL: flat_atomic_inc_ret_i32: ; CI-LABEL: flat_atomic_inc_ret_i32:
; GCN: ; %bb.0: ; CI: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GCN-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; GCN-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; GCN-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; GCN-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CI-NEXT: buffer_wbinvl1_vol
; GCN-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: s_endpgm ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_inc_ret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_inc_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: flat_store_dword v[0:1], v2
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_inc_ret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: flat_store_dword v[0:1], v2
; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_ret_i32: ; GFX11-LABEL: flat_atomic_inc_ret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 glc ; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 glc
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: flat_store_b32 v[0:1], v2
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %ptr, i32 42 seq_cst, align 4
store i32 %result, ptr %out store i32 %result, ptr %out, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_ret_i32_offset: ; CI-LABEL: flat_atomic_inc_ret_i32_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 16 ; CI-NEXT: s_add_u32 s2, s2, 16
; CI-NEXT: s_addc_u32 s3, s3, 0 ; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_ret_i32_offset: ; VI-LABEL: flat_atomic_inc_ret_i32_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 16 ; VI-NEXT: s_add_u32 s2, s2, 16
; VI-NEXT: s_addc_u32 s3, s3, 0 ; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_ret_i32_offset: ; GFX9-LABEL: flat_atomic_inc_ret_i32_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42 ; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc ; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_store_dword v[0:1], v2 ; GFX9-NEXT: flat_store_dword v[0:1], v2
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_ret_i32_offset: ; GFX10-LABEL: flat_atomic_inc_ret_i32_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42 ; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s2, s2, 16 ; GFX10-NEXT: s_add_u32 s2, s2, 16
; GFX10-NEXT: s_addc_u32 s3, s3, 0 ; GFX10-NEXT: s_addc_u32 s3, s3, 0
; GFX10-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_store_dword v[0:1], v2 ; GFX10-NEXT: flat_store_dword v[0:1], v2
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_ret_i32_offset: ; GFX11-LABEL: flat_atomic_inc_ret_i32_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc ; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: flat_store_b32 v[0:1], v2
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, ptr %ptr, i32 4 %gep = getelementptr i32, ptr %ptr, i32 4
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4
store i32 %result, ptr %out store i32 %result, ptr %out, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) nounwind { define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {
; GCN-LABEL: flat_atomic_inc_noret_i32: ; CI-LABEL: flat_atomic_inc_noret_i32:
; GCN: ; %bb.0: ; CI: ; %bb.0:
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GCN-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; GCN-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: flat_atomic_inc v[0:1], v2 ; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_endpgm ; CI-NEXT: flat_atomic_inc v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_inc_noret_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_inc_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc v[0:1], v2
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_inc_noret_i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_noret_i32: ; GFX11-LABEL: flat_atomic_inc_noret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %ptr, i32 42 seq_cst, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) nounwind { define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_noret_i32_offset: ; CI-LABEL: flat_atomic_inc_noret_i32_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 16 ; CI-NEXT: s_add_u32 s0, s0, 16
; CI-NEXT: s_addc_u32 s1, s1, 0 ; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v[0:1], v2 ; CI-NEXT: flat_atomic_inc v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_noret_i32_offset: ; VI-LABEL: flat_atomic_inc_noret_i32_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 16 ; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0 ; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v[0:1], v2 ; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_noret_i32_offset: ; GFX9-LABEL: flat_atomic_inc_noret_i32_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 42 ; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:16 ; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_noret_i32_offset: ; GFX10-LABEL: flat_atomic_inc_noret_i32_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v2, 42 ; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s0, s0, 16 ; GFX10-NEXT: s_add_u32 s0, s0, 16
; GFX10-NEXT: s_addc_u32 s1, s1, 0 ; GFX10-NEXT: s_addc_u32 s1, s1, 0
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc v[0:1], v2 ; GFX10-NEXT: flat_atomic_inc v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_noret_i32_offset: ; GFX11-LABEL: flat_atomic_inc_noret_i32_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16 ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i32, ptr %ptr, i32 4 %gep = getelementptr i32, ptr %ptr, i32 4
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_ret_i32_offset_addr64: ; CI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: v_mov_b32_e32 v3, 42 ; CI-NEXT: v_mov_b32_e32 v3, 42
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0 ; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc ; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_store_dword v[0:1], v3 ; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_ret_i32_offset_addr64: ; VI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: v_mov_b32_e32 v3, 42 ; VI-NEXT: v_mov_b32_e32 v3, 42
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0 ; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc ; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v3 ; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_addr64: ; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 42 ; GFX9-NEXT: v_mov_b32_e32 v3, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2 ; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc v3, v[0:1], v3 offset:20 glc ; GFX9-NEXT: flat_atomic_inc v3, v[0:1], v3 offset:20 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_store_dword v[0:1], v3 ; GFX9-NEXT: flat_store_dword v[0:1], v3
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_ret_i32_offset_addr64: ; GFX10-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, 42 ; GFX10-NEXT: v_mov_b32_e32 v3, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc v3, v[0:1], v3 glc ; GFX10-NEXT: flat_atomic_inc v3, v[0:1], v3 glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_store_dword v[0:1], v3 ; GFX10-NEXT: flat_store_dword v[0:1], v3
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_ret_i32_offset_addr64: ; GFX11-LABEL: flat_atomic_inc_ret_i32_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_dual_mov_b32 v3, 42 :: v_dual_lshlrev_b32 v2, 2, v0 ; GFX11-NEXT: v_dual_mov_b32 v3, 42 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u32 v3, v[0:1], v3 offset:20 glc ; GFX11-NEXT: flat_atomic_inc_u32 v3, v[0:1], v3 offset:20 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_store_b32 v[0:1], v3 ; GFX11-NEXT: flat_store_b32 v[0:1], v3
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, ptr %ptr, i32 %id %gep.tid = getelementptr i32, ptr %ptr, i32 %id
%out.gep = getelementptr i32, ptr %out, i32 %id %out.gep = getelementptr i32, ptr %out, i32 %id
%gep = getelementptr i32, ptr %gep.tid, i32 5 %gep = getelementptr i32, ptr %gep.tid, i32 5
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4
store i32 %result, ptr %out.gep store i32 %result, ptr %out.gep, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_noret_i32_offset_addr64: ; CI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0 ; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v2, 42 ; CI-NEXT: v_mov_b32_e32 v2, 42
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc v[0:1], v2 ; CI-NEXT: flat_atomic_inc v[0:1], v2
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_noret_i32_offset_addr64: ; VI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0 ; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v2, 42 ; VI-NEXT: v_mov_b32_e32 v2, 42
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc v[0:1], v2 ; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_addr64: ; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_mov_b32_e32 v2, 42 ; GFX9-NEXT: v_mov_b32_e32 v2, 42
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:20 ; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:20
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_noret_i32_offset_addr64: ; GFX10-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v2, 42 ; GFX10-NEXT: v_mov_b32_e32 v2, 42
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 20
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc v[0:1], v2 ; GFX10-NEXT: flat_atomic_inc v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_addr64: ; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:20 ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:20
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i32, ptr %ptr, i32 %id %gep.tid = getelementptr i32, ptr %ptr, i32 %id
%gep = getelementptr i32, ptr %gep.tid, i32 5 %gep = getelementptr i32, ptr %gep.tid, i32 5
%result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4
ret void ret void
} }
@lds1 = internal addrspace(3) global [512 x i64] undef, align 8 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #1 {
define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {
; CI-LABEL: atomic_inc_shl_base_lds_0_i64: ; CI-LABEL: atomic_inc_shl_base_lds_0_i64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v1, 9 ; CI-NEXT: v_mov_b32_e32 v1, 9
; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v0 ; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; CI-NEXT: v_mov_b32_e32 v2, 0 ; CI-NEXT: v_mov_b32_e32 v2, 0
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16 ; CI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v4, s3 ; CI-NEXT: v_mov_b32_e32 v4, s3
; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v0 ; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v0
; CI-NEXT: v_mov_b32_e32 v3, s2 ; CI-NEXT: v_mov_b32_e32 v3, s2
; CI-NEXT: flat_store_dword v[3:4], v0 ; CI-NEXT: flat_store_dword v[3:4], v0
; CI-NEXT: v_mov_b32_e32 v4, s1 ; CI-NEXT: v_mov_b32_e32 v4, s1
; CI-NEXT: v_mov_b32_e32 v3, s0 ; CI-NEXT: v_mov_b32_e32 v3, s0
; CI-NEXT: flat_store_dwordx2 v[3:4], v[1:2] ; CI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: atomic_inc_shl_base_lds_0_i64: ; VI-LABEL: atomic_inc_shl_base_lds_0_i64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v1, 9 ; VI-NEXT: v_mov_b32_e32 v1, 9
; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v0 ; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; VI-NEXT: v_mov_b32_e32 v2, 0 ; VI-NEXT: v_mov_b32_e32 v2, 0
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16 ; VI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, s3 ; VI-NEXT: v_mov_b32_e32 v4, s3
; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v0 ; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v0
; VI-NEXT: v_mov_b32_e32 v3, s2 ; VI-NEXT: v_mov_b32_e32 v3, s2
; VI-NEXT: flat_store_dword v[3:4], v0 ; VI-NEXT: flat_store_dword v[3:4], v0
; VI-NEXT: v_mov_b32_e32 v4, s1 ; VI-NEXT: v_mov_b32_e32 v4, s1
; VI-NEXT: v_mov_b32_e32 v3, s0 ; VI-NEXT: v_mov_b32_e32 v3, s0
; VI-NEXT: flat_store_dwordx2 v[3:4], v[1:2] ; VI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64: ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v1, 9 ; GFX9-NEXT: v_mov_b32_e32 v1, 9
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16 ; GFX9-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, 2, v0 ; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0 ; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v3, v0, s[2:3] ; GFX9-NEXT: global_store_dword v3, v0, s[2:3]
; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1] ; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: atomic_inc_shl_base_lds_0_i64: ; GFX10-LABEL: atomic_inc_shl_base_lds_0_i64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: v_mov_b32_e32 v1, 9 ; GFX10-NEXT: v_mov_b32_e32 v1, 9
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16 ; GFX10-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: global_store_dword v3, v0, s[2:3] ; GFX10-NEXT: global_store_dword v3, v0, s[2:3]
; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1] ; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: atomic_inc_shl_base_lds_0_i64: ; GFX11-LABEL: atomic_inc_shl_base_lds_0_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: v_mov_b32_e32 v1, 9 ; GFX11-NEXT: v_mov_b32_e32 v1, 9
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 3, v0 ; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 3, v0
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0 ; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16 ; GFX11-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:16
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v3, v0, s[2:3] ; GFX11-NEXT: global_store_b32 v3, v0, s[2:3]
; GFX11-NEXT: global_store_b64 v3, v[1:2], s[0:1] ; GFX11-NEXT: global_store_b64 v3, v[1:2], s[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #2
%idx.0 = add nsw i32 %tid.x, 2 %idx.0 = add nsw i32 %tid.x, 2
%arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.0 %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.0
%val0 = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %arrayidx0, i64 9, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr addrspace(3) %arrayidx0, i64 9 seq_cst, align 8
store i32 %idx.0, ptr addrspace(1) %add_use store i32 %idx.0, ptr addrspace(1) %add_use, align 4
store i64 %val0, ptr addrspace(1) %out store i64 %result, ptr addrspace(1) %out, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {
; GCN-LABEL: flat_atomic_inc_ret_i64: ; CI-LABEL: flat_atomic_inc_ret_i64:
; GCN: ; %bb.0: ; CI: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GCN-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; GCN-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: v_mov_b32_e32 v2, s2
; GCN-NEXT: v_mov_b32_e32 v3, s3 ; CI-NEXT: v_mov_b32_e32 v3, s3
; GCN-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GCN-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CI-NEXT: buffer_wbinvl1_vol
; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: v_mov_b32_e32 v3, s1
; GCN-NEXT: s_endpgm ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_inc_ret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_inc_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_inc_ret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_ret_i64: ; GFX11-LABEL: flat_atomic_inc_ret_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2
; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: v_mov_b32_e32 v3, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] glc ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] glc
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %ptr, i64 42 seq_cst, align 8
store i64 %result, ptr %out store i64 %result, ptr %out, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_ret_i64_offset: ; CI-LABEL: flat_atomic_inc_ret_i64_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s2, s2, 32 ; CI-NEXT: s_add_u32 s2, s2, 32
; CI-NEXT: s_addc_u32 s3, s3, 0 ; CI-NEXT: s_addc_u32 s3, s3, 0
; CI-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: v_mov_b32_e32 v3, s3 ; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_ret_i64_offset: ; VI-LABEL: flat_atomic_inc_ret_i64_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s2, s2, 32 ; VI-NEXT: s_add_u32 s2, s2, 32
; VI-NEXT: s_addc_u32 s3, s3, 0 ; VI-NEXT: s_addc_u32 s3, s3, 0
; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3 ; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_ret_i64_offset: ; GFX9-LABEL: flat_atomic_inc_ret_i64_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s2 ; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: v_mov_b32_e32 v3, s3 ; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_ret_i64_offset: ; GFX10-LABEL: flat_atomic_inc_ret_i64_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s2, s2, 32 ; GFX10-NEXT: s_add_u32 s2, s2, 32
; GFX10-NEXT: s_addc_u32 s3, s3, 0 ; GFX10-NEXT: s_addc_u32 s3, s3, 0
; GFX10-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_ret_i64_offset: ; GFX11-LABEL: flat_atomic_inc_ret_i64_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2 ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2
; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: v_mov_b32_e32 v3, s3
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 glc ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 glc
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, ptr %ptr, i32 4 %gep = getelementptr i64, ptr %ptr, i32 4
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8
store i64 %result, ptr %out store i64 %result, ptr %out, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) nounwind { define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {
; GCN-LABEL: flat_atomic_inc_noret_i64: ; CI-LABEL: flat_atomic_inc_noret_i64:
; GCN: ; %bb.0: ; CI: ; %bb.0:
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GCN-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; GCN-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; GCN-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_endpgm ; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm
;
; VI-LABEL: flat_atomic_inc_noret_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: flat_atomic_inc_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: flat_atomic_inc_noret_i64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_noret_i64: ; GFX11-LABEL: flat_atomic_inc_noret_i64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] ; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %ptr, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) nounwind { define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_noret_i64_offset: ; CI-LABEL: flat_atomic_inc_noret_i64_offset:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s0, s0, 32 ; CI-NEXT: s_add_u32 s0, s0, 32
; CI-NEXT: s_addc_u32 s1, s1, 0 ; CI-NEXT: s_addc_u32 s1, s1, 0
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_noret_i64_offset: ; VI-LABEL: flat_atomic_inc_noret_i64_offset:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 32 ; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0 ; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_noret_i64_offset: ; GFX9-LABEL: flat_atomic_inc_noret_i64_offset:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:32 ; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_noret_i64_offset: ; GFX10-LABEL: flat_atomic_inc_noret_i64_offset:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_u32 s0, s0, 32 ; GFX10-NEXT: s_add_u32 s0, s0, 32
; GFX10-NEXT: s_addc_u32 s1, s1, 0 ; GFX10-NEXT: s_addc_u32 s1, s1, 0
; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_noret_i64_offset: ; GFX11-LABEL: flat_atomic_inc_noret_i64_offset:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 42 ; GFX11-NEXT: v_mov_b32_e32 v0, 42
; GFX11-NEXT: v_mov_b32_e32 v1, 0 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:32 ; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:32
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%gep = getelementptr i64, ptr %ptr, i32 4 %gep = getelementptr i64, ptr %ptr, i32 4
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_ret_i64_offset_addr64: ; CI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: v_mov_b32_e32 v1, s3 ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v4 ; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2 ; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: v_mov_b32_e32 v3, s1 ; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v2, s0 ; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_ret_i64_offset_addr64: ; VI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v4 ; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2 ; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_addr64: ; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s3 ; GFX9-NEXT: v_mov_b32_e32 v4, s3
; GFX9-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[3:4], v[1:2] offset:40 glc ; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[3:4], v[1:2] offset:40 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5 ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_ret_i64_offset_addr64: ; GFX10-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v4 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_ret_i64_offset_addr64: ; GFX11-LABEL: flat_atomic_inc_ret_i64_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2 ; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s3 ; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4 ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] offset:40 glc ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] offset:40 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 ; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, ptr %ptr, i32 %id %gep.tid = getelementptr i64, ptr %ptr, i32 %id
%out.gep = getelementptr i64, ptr %out, i32 %id %out.gep = getelementptr i64, ptr %out, i32 %id
%gep = getelementptr i64, ptr %gep.tid, i32 5 %gep = getelementptr i64, ptr %gep.tid, i32 5
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8
store i64 %result, ptr %out.gep store i64 %result, ptr %out.gep, align 4
ret void ret void
} }
define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #0 { define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1 {
; CI-LABEL: flat_atomic_inc_noret_i64_offset_addr64: ; CI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0 ; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2 ; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2 ; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v2
; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_wbinvl1_vol
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: flat_atomic_inc_noret_i64_offset_addr64: ; VI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2 ; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2 ; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2
; VI-NEXT: v_mov_b32_e32 v1, 0 ; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_addr64: ; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s1 ; GFX9-NEXT: v_mov_b32_e32 v4, s1
; GFX9-NEXT: v_mov_b32_e32 v3, s0 ; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v0 ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v0
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_atomic_inc_x2 v[3:4], v[1:2] offset:40 ; GFX9-NEXT: flat_atomic_inc_x2 v[3:4], v[1:2] offset:40
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: flat_atomic_inc_noret_i64_offset_addr64: ; GFX10-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v2 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v2
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 40
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_addr64: ; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_addr64:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 42 ; GFX11-NEXT: v_mov_b32_e32 v2, 42
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s0 ; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s0
; GFX11-NEXT: v_mov_b32_e32 v1, s1 ; GFX11-NEXT: v_mov_b32_e32 v1, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4 ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:40 ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:40
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%id = call i32 @llvm.amdgcn.workitem.id.x() %id = call i32 @llvm.amdgcn.workitem.id.x()
%gep.tid = getelementptr i64, ptr %ptr, i32 %id %gep.tid = getelementptr i64, ptr %ptr, i32 %id
%gep = getelementptr i64, ptr %gep.tid, i32 5 %gep = getelementptr i64, ptr %gep.tid, i32 5
%result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false) %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8
ret void ret void
} }
define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(3) %ptr) #0 { define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: nocse_lds_atomic_inc_ret_i32: ; CI-LABEL: nocse_lds_atomic_inc_ret_i32:
; CI: ; %bb.0: ; CI: ; %bb.0:
; CI-NEXT: s_load_dword s6, s[4:5], 0x4 ; CI-NEXT: s_load_dword s6, s[4:5], 0x4
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42 ; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s6 ; CI-NEXT: v_mov_b32_e32 v1, s6
; CI-NEXT: ds_inc_rtn_u32 v4, v1, v0 ; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u32 v5, v1, v0 ; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: ds_inc_rtn_u32 v3, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0 ; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: v_mov_b32_e32 v2, s2 ; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: v_mov_b32_e32 v3, s3 ; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: s_waitcnt lgkmcnt(1) ; CI-NEXT: v_mov_b32_e32 v1, s3
; CI-NEXT: flat_store_dword v[0:1], v4 ; CI-NEXT: flat_store_dword v[0:1], v3
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: flat_store_dword v[2:3], v5
; CI-NEXT: s_endpgm ; CI-NEXT: s_endpgm
; ;
; VI-LABEL: nocse_lds_atomic_inc_ret_i32: ; VI-LABEL: nocse_lds_atomic_inc_ret_i32:
; VI: ; %bb.0: ; VI: ; %bb.0:
; VI-NEXT: s_load_dword s6, s[4:5], 0x10 ; VI-NEXT: s_load_dword s6, s[4:5], 0x10
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42 ; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1 ; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s6 ; VI-NEXT: v_mov_b32_e32 v1, s6
; VI-NEXT: ds_inc_rtn_u32 v4, v1, v0 ; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u32 v5, v1, v0 ; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_inc_rtn_u32 v3, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2 ; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: v_mov_b32_e32 v3, s3 ; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: s_waitcnt lgkmcnt(1) ; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: flat_store_dword v[0:1], v4 ; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dword v[2:3], v5
; VI-NEXT: s_endpgm ; VI-NEXT: s_endpgm
; ;
; GFX9-LABEL: nocse_lds_atomic_inc_ret_i32: ; GFX9-LABEL: nocse_lds_atomic_inc_ret_i32:
; GFX9: ; %bb.0: ; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x10 ; GFX9-NEXT: s_load_dword s6, s[4:5], 0x10
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42 ; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u32 v2, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: ds_inc_rtn_u32 v2, v1, v0
; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NEXT: global_store_dword v1, v2, s[0:1] ; GFX9-NEXT: global_store_dword v1, v2, s[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v1, v0, s[2:3] ; GFX9-NEXT: global_store_dword v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm ; GFX9-NEXT: s_endpgm
; ;
; GFX10-LABEL: nocse_lds_atomic_inc_ret_i32: ; GFX10-LABEL: nocse_lds_atomic_inc_ret_i32:
; GFX10: ; %bb.0: ; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[4:5], 0x10 ; GFX10-NEXT: s_load_dword s0, s[4:5], 0x10
; GFX10-NEXT: v_mov_b32_e32 v0, 42 ; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s0 ; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: ds_inc_rtn_u32 v2, v1, v0 ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_inc_rtn_u32 v0, v1, v0 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: ds_inc_rtn_u32 v2, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: global_store_dword v1, v2, s[0:1] ; GFX10-NEXT: global_store_dword v1, v2, s[0:1]
; GFX10-NEXT: global_store_dword v1, v0, s[2:3] ; GFX10-NEXT: global_store_dword v1, v0, s[2:3]
; GFX10-NEXT: s_endpgm ; GFX10-NEXT: s_endpgm
; ;
; GFX11-LABEL: nocse_lds_atomic_inc_ret_i32: ; GFX11-LABEL: nocse_lds_atomic_inc_ret_i32:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x10 ; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s2 ; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
; GFX11-NEXT: ds_inc_rtn_u32 v2, v1, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_inc_rtn_u32 v0, v1, v0 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: v_mov_b32_e32 v1, 0 ; GFX11-NEXT: ds_inc_rtn_u32 v2, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v1, v2, s[0:1] ; GFX11-NEXT: global_store_b32 v1, v2, s[0:1]
; GFX11-NEXT: global_store_b32 v1, v0, s[2:3] ; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false) %result0 = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false) %result1 = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 seq_cst, align 4
store i32 %result0, ptr addrspace(1) %out0, align 4
store i32 %result0, ptr addrspace(1) %out0 store i32 %result1, ptr addrspace(1) %out1, align 4
store i32 %result1, ptr addrspace(1) %out1
ret void ret void
} }
attributes #0 = { nounwind } attributes #0 = { nounwind speculatable willreturn memory(none) }
attributes #1 = { nounwind readnone } attributes #1 = { nounwind }
attributes #2 = { nounwind argmemonly } attributes #2 = { nounwind memory(none) }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll

  • This file was moved to llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll.

llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll

Show First 20 Lines • Show All 625 Lines • ▼ Show 20 Linesendif:
%x = phi ptr addrspace(3) [ %tmp1, %if ], [ null, %entry ] %x = phi ptr addrspace(3) [ %tmp1, %if ], [ null, %entry ]
store ptr addrspace(3) %x, ptr addrspace(3) %out.gep store ptr addrspace(3) %x, ptr addrspace(3) %out.gep
br label %done br label %done
done: done:
ret void ret void
} }
; OPT-LABEL: @test_sink_local_small_offset_atomic_inc_i32(
; OPT: %sunkaddr = getelementptr i8, ptr addrspace(3) %in, i32 28
; OPT: %tmp1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %sunkaddr, i32 2, i32 0, i32 0, i1 false)
define amdgpu_kernel void @test_sink_local_small_offset_atomic_inc_i32(ptr addrspace(3) %out, ptr addrspace(3) %in) {
entry:
%out.gep = getelementptr i32, ptr addrspace(3) %out, i32 999999
%in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7
%tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
%tmp0 = icmp eq i32 %tid, 0
br i1 %tmp0, label %endif, label %if
if:
%tmp1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %in.gep, i32 2, i32 0, i32 0, i1 false)
br label %endif
endif:
%x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
store i32 %x, ptr addrspace(3) %out.gep
br label %done
done:
ret void
}
; OPT-LABEL: @test_sink_local_small_offset_atomic_dec_i32(
; OPT: %sunkaddr = getelementptr i8, ptr addrspace(3) %in, i32 28
; OPT: %tmp1 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %sunkaddr, i32 2, i32 0, i32 0, i1 false)
define amdgpu_kernel void @test_sink_local_small_offset_atomic_dec_i32(ptr addrspace(3) %out, ptr addrspace(3) %in) {
entry:
%out.gep = getelementptr i32, ptr addrspace(3) %out, i32 999999
%in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7
%tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
%tmp0 = icmp eq i32 %tid, 0
br i1 %tmp0, label %endif, label %if
if:
%tmp1 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %in.gep, i32 2, i32 0, i32 0, i1 false)
br label %endif
endif:
%x = phi i32 [ %tmp1, %if ], [ 0, %entry ]
store i32 %x, ptr addrspace(3) %out.gep
br label %done
done:
ret void
}
; OPT-LABEL: @test_sink_global_small_min_scratch_global_offset( ; OPT-LABEL: @test_sink_global_small_min_scratch_global_offset(
; OPT-SICIVI: %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 -4096 ; OPT-SICIVI: %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 -4096
; OPT-SICIV: br ; OPT-SICIV: br
; OPT-SICIVI: %tmp1 = load i8, ptr addrspace(1) %in.gep ; OPT-SICIVI: %tmp1 = load i8, ptr addrspace(1) %in.gep
; OPT-GFX9: br ; OPT-GFX9: br
; OPT-GFX9: %sunkaddr = getelementptr i8, ptr addrspace(1) %in, i64 -4096 ; OPT-GFX9: %sunkaddr = getelementptr i8, ptr addrspace(1) %in, i64 -4096
; OPT-GFX9: load i8, ptr addrspace(1) %sunkaddr ; OPT-GFX9: load i8, ptr addrspace(1) %sunkaddr
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Linesendif:
store i32 %x, ptr addrspace(3) %out.gep store i32 %x, ptr addrspace(3) %out.gep
br label %done br label %done
done: done:
ret void ret void
} }
declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
declare i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1) #2
declare i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1) #2
declare i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) nocapture, i1 immarg) #3 declare i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) nocapture, i1 immarg) #3
declare i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) nocapture, i1 immarg) #3 declare i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) nocapture, i1 immarg) #3
attributes #0 = { nounwind readnone } attributes #0 = { nounwind readnone }
attributes #1 = { nounwind } attributes #1 = { nounwind }
attributes #2 = { nounwind argmemonly } attributes #2 = { nounwind argmemonly }
attributes #3 = { argmemonly convergent nounwind willreturn } attributes #3 = { argmemonly convergent nounwind willreturn }

llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll

Show First 20 Lines • Show All 424 Lines • ▼ Show 20 Lines; CHECK-NEXT: s_endpgm
ret void ret void
} }
define protected amdgpu_kernel void @inc(ptr addrspace(1) %p, ptr addrspace(1) %q) { define protected amdgpu_kernel void @inc(ptr addrspace(1) %p, ptr addrspace(1) %q) {
; CHECK-LABEL: inc: ; CHECK-LABEL: inc:
; CHECK: ; %bb.0: ; CHECK: ; %bb.0:
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; CHECK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0 ; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_atomic_inc v2, v0, v0, s[0:1] glc ; CHECK-NEXT: global_atomic_inc v2, v0, v1, s[0:1] glc
; CHECK-NEXT: v_mov_b32_e32 v0, s2 ; CHECK-NEXT: v_mov_b32_e32 v0, s2
; CHECK-NEXT: v_mov_b32_e32 v1, s3 ; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1] ; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
; CHECK-NEXT: v_mov_b32_e32 v2, 1.0 ; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: global_store_dword v[0:1], v2, off ; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm ; CHECK-NEXT: s_endpgm
%n32 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %p, i32 0, i32 0, i32 0, i1 false) %n32 = atomicrmw uinc_wrap ptr addrspace(1) %p, i32 1 monotonic
%n64 = zext i32 %n32 to i64 %n64 = zext i32 %n32 to i64
%p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0 %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
store float 1.0, ptr addrspace(1) %p1 store float 1.0, ptr addrspace(1) %p1
ret void ret void
} }
define protected amdgpu_kernel void @dec(ptr addrspace(1) %p, ptr addrspace(1) %q) { define protected amdgpu_kernel void @dec(ptr addrspace(1) %p, ptr addrspace(1) %q) {
; CHECK-LABEL: dec: ; CHECK-LABEL: dec:
; CHECK: ; %bb.0: ; CHECK: ; %bb.0:
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; CHECK-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; CHECK-NEXT: v_mov_b32_e32 v0, 0 ; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 1
; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_atomic_dec v2, v0, v0, s[0:1] glc ; CHECK-NEXT: global_atomic_dec v2, v0, v1, s[0:1] glc
; CHECK-NEXT: v_mov_b32_e32 v0, s2 ; CHECK-NEXT: v_mov_b32_e32 v0, s2
; CHECK-NEXT: v_mov_b32_e32 v1, s3 ; CHECK-NEXT: v_mov_b32_e32 v1, s3
; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1] ; CHECK-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, 12, v[0:1]
; CHECK-NEXT: v_mov_b32_e32 v2, 1.0 ; CHECK-NEXT: v_mov_b32_e32 v2, 1.0
; CHECK-NEXT: global_store_dword v[0:1], v2, off ; CHECK-NEXT: global_store_dword v[0:1], v2, off
; CHECK-NEXT: s_endpgm ; CHECK-NEXT: s_endpgm
%n32 = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %p, i32 0, i32 0, i32 0, i1 false) %n32 = atomicrmw udec_wrap ptr addrspace(1) %p, i32 1 monotonic
%n64 = zext i32 %n32 to i64 %n64 = zext i32 %n32 to i64
%p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0 %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
store float 1.0, ptr addrspace(1) %p1 store float 1.0, ptr addrspace(1) %p1
ret void ret void
} }
define protected amdgpu_kernel void @fadd(ptr addrspace(1) %p, ptr addrspace(1) %q) { define protected amdgpu_kernel void @fadd(ptr addrspace(1) %p, ptr addrspace(1) %q) {
; CHECK-LABEL: fadd: ; CHECK-LABEL: fadd:
▲ Show 20 Lines • Show All 475 Lines • ▼ Show 20 Lines; CHECK-NEXT: s_endpgm
%f64 = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double 1.0, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0) %f64 = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double 1.0, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
%n32 = fptoui double %f64 to i32 %n32 = fptoui double %f64 to i32
%n64 = zext i32 %n32 to i64 %n64 = zext i32 %n32 to i64
%p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0 %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
store float 1.0, ptr addrspace(1) %p1 store float 1.0, ptr addrspace(1) %p1
ret void ret void
} }
declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1), i32, i32 immarg, i32 immarg, i1 immarg)
declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1), i32, i32 immarg, i32 immarg, i1 immarg)
declare double @llvm.amdgcn.global.atomic.fmin.f64.p1.f64(ptr addrspace(1), double) declare double @llvm.amdgcn.global.atomic.fmin.f64.p1.f64(ptr addrspace(1), double)
declare double @llvm.amdgcn.global.atomic.fmax.f64.p1.f64(ptr addrspace(1), double) declare double @llvm.amdgcn.global.atomic.fmax.f64.p1.f64(ptr addrspace(1), double)
declare i32 @llvm.amdgcn.raw.buffer.atomic.swap.i32(i32, <4 x i32>, i32, i32, i32) declare i32 @llvm.amdgcn.raw.buffer.atomic.swap.i32(i32, <4 x i32>, i32, i32, i32)
declare i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i32) declare i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32, <4 x i32>, i32, i32, i32)
declare i32 @llvm.amdgcn.raw.buffer.atomic.sub.i32(i32, <4 x i32>, i32, i32, i32) declare i32 @llvm.amdgcn.raw.buffer.atomic.sub.i32(i32, <4 x i32>, i32, i32, i32)
declare i32 @llvm.amdgcn.raw.buffer.atomic.smin.i32(i32, <4 x i32>, i32, i32, i32) declare i32 @llvm.amdgcn.raw.buffer.atomic.smin.i32(i32, <4 x i32>, i32, i32, i32)
declare i32 @llvm.amdgcn.raw.buffer.atomic.smax.i32(i32, <4 x i32>, i32, i32, i32) declare i32 @llvm.amdgcn.raw.buffer.atomic.smax.i32(i32, <4 x i32>, i32, i32, i32)
declare i32 @llvm.amdgcn.raw.buffer.atomic.umin.i32(i32, <4 x i32>, i32, i32, i32) declare i32 @llvm.amdgcn.raw.buffer.atomic.umin.i32(i32, <4 x i32>, i32, i32, i32)
Show All 10 Lines

llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll

Show First 20 Lines • Show All 3,354 Lines • ▼ Show 20 Lines; GFX11-NEXT: s_endpgm
%unused = cmpxchg ptr addrspace(1) %gep1, i64 %cmp, i64 %data seq_cst seq_cst %unused = cmpxchg ptr addrspace(1) %gep1, i64 %cmp, i64 %data seq_cst seq_cst
ret void ret void
} }
; -------------------------------------------------------------------------------- ; --------------------------------------------------------------------------------
; amdgcn atomic inc ; amdgcn atomic inc
; -------------------------------------------------------------------------------- ; --------------------------------------------------------------------------------
declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
define amdgpu_ps float @global_inc_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps float @global_inc_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_inc_saddr_i32_rtn: ; GCN-LABEL: global_inc_saddr_i32_rtn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc v0, v0, v1, s[2:3] glc ; GCN-NEXT: global_atomic_inc v0, v0, v1, s[2:3] glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_inc_saddr_i32_rtn: ; GFX11-LABEL: global_inc_saddr_i32_rtn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: global_atomic_inc_u32 v0, v0, v1, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%rtn = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep0, i32 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i32 %data monotonic
%cast.rtn = bitcast i32 %rtn to float %cast.rtn = bitcast i32 %rtn to float
ret float %cast.rtn ret float %cast.rtn
} }
define amdgpu_ps float @global_inc_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps float @global_inc_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_inc_saddr_i32_rtn_neg128: ; GCN-LABEL: global_inc_saddr_i32_rtn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc v0, v0, v1, s[2:3] offset:-128 glc ; GCN-NEXT: global_atomic_inc v0, v0, v1, s[2:3] offset:-128 glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_inc_saddr_i32_rtn_neg128: ; GFX11-LABEL: global_inc_saddr_i32_rtn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: global_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%rtn = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep1, i32 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i32 %data monotonic
%cast.rtn = bitcast i32 %rtn to float %cast.rtn = bitcast i32 %rtn to float
ret float %cast.rtn ret float %cast.rtn
} }
define amdgpu_ps void @global_inc_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps void @global_inc_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_inc_saddr_i32_nortn: ; GCN-LABEL: global_inc_saddr_i32_nortn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc v0, v1, s[2:3] ; GCN-NEXT: global_atomic_inc v0, v1, s[2:3]
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_inc_saddr_i32_nortn: ; GFX11-LABEL: global_inc_saddr_i32_nortn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[2:3] ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[2:3]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%unused = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep0, i32 %data, i32 0, i32 0, i1 false) %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i32 %data monotonic
ret void ret void
} }
define amdgpu_ps void @global_inc_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps void @global_inc_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_inc_saddr_i32_nortn_neg128: ; GCN-LABEL: global_inc_saddr_i32_nortn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc v0, v1, s[2:3] offset:-128 ; GCN-NEXT: global_atomic_inc v0, v1, s[2:3] offset:-128
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_inc_saddr_i32_nortn_neg128: ; GFX11-LABEL: global_inc_saddr_i32_nortn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[2:3] offset:-128
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%unused = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep1, i32 %data, i32 0, i32 0, i1 false) %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i32 %data monotonic
ret void ret void
} }
define amdgpu_ps <2 x float> @global_inc_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps <2 x float> @global_inc_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_inc_saddr_i64_rtn: ; GCN-LABEL: global_inc_saddr_i64_rtn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GCN-NEXT: global_atomic_inc_x2 v[0:1], v0, v[1:2], s[2:3] glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_inc_saddr_i64_rtn: ; GFX11-LABEL: global_inc_saddr_i64_rtn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v0, v[1:2], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%rtn = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep0, i64 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i64 %data monotonic
%cast.rtn = bitcast i64 %rtn to <2 x float> %cast.rtn = bitcast i64 %rtn to <2 x float>
ret <2 x float> %cast.rtn ret <2 x float> %cast.rtn
} }
define amdgpu_ps <2 x float> @global_inc_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps <2 x float> @global_inc_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_inc_saddr_i64_rtn_neg128: ; GCN-LABEL: global_inc_saddr_i64_rtn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GCN-NEXT: global_atomic_inc_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_inc_saddr_i64_rtn_neg128: ; GFX11-LABEL: global_inc_saddr_i64_rtn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%rtn = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep1, i64 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i64 %data monotonic
%cast.rtn = bitcast i64 %rtn to <2 x float> %cast.rtn = bitcast i64 %rtn to <2 x float>
ret <2 x float> %cast.rtn ret <2 x float> %cast.rtn
} }
define amdgpu_ps void @global_inc_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps void @global_inc_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_inc_saddr_i64_nortn: ; GCN-LABEL: global_inc_saddr_i64_nortn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc_x2 v0, v[1:2], s[2:3] ; GCN-NEXT: global_atomic_inc_x2 v0, v[1:2], s[2:3]
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_inc_saddr_i64_nortn: ; GFX11-LABEL: global_inc_saddr_i64_nortn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[2:3] ; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[2:3]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%unused = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep0, i64 %data, i32 0, i32 0, i1 false) %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i64 %data monotonic
ret void ret void
} }
define amdgpu_ps void @global_inc_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps void @global_inc_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_inc_saddr_i64_nortn_neg128: ; GCN-LABEL: global_inc_saddr_i64_nortn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_inc_x2 v0, v[1:2], s[2:3] offset:-128 ; GCN-NEXT: global_atomic_inc_x2 v0, v[1:2], s[2:3] offset:-128
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_inc_saddr_i64_nortn_neg128: ; GFX11-LABEL: global_inc_saddr_i64_nortn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[2:3] offset:-128
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%unused = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep1, i64 %data, i32 0, i32 0, i1 false) %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i64 %data monotonic
ret void ret void
} }
; -------------------------------------------------------------------------------- ; --------------------------------------------------------------------------------
; amdgcn atomic dec ; amdgcn atomic dec
; -------------------------------------------------------------------------------- ; --------------------------------------------------------------------------------
declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) nocapture, i32, i32 immarg, i32 immarg, i1 immarg) #0
declare i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) nocapture, i64, i32 immarg, i32 immarg, i1 immarg) #0
define amdgpu_ps float @global_dec_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps float @global_dec_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_dec_saddr_i32_rtn: ; GCN-LABEL: global_dec_saddr_i32_rtn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec v0, v0, v1, s[2:3] glc ; GCN-NEXT: global_atomic_dec v0, v0, v1, s[2:3] glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_dec_saddr_i32_rtn: ; GFX11-LABEL: global_dec_saddr_i32_rtn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: global_atomic_dec_u32 v0, v0, v1, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%rtn = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep0, i32 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep0, i32 %data monotonic
%cast.rtn = bitcast i32 %rtn to float %cast.rtn = bitcast i32 %rtn to float
ret float %cast.rtn ret float %cast.rtn
} }
define amdgpu_ps float @global_dec_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps float @global_dec_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_dec_saddr_i32_rtn_neg128: ; GCN-LABEL: global_dec_saddr_i32_rtn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec v0, v0, v1, s[2:3] offset:-128 glc ; GCN-NEXT: global_atomic_dec v0, v0, v1, s[2:3] offset:-128 glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_dec_saddr_i32_rtn_neg128: ; GFX11-LABEL: global_dec_saddr_i32_rtn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: global_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%rtn = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep1, i32 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep1, i32 %data monotonic
%cast.rtn = bitcast i32 %rtn to float %cast.rtn = bitcast i32 %rtn to float
ret float %cast.rtn ret float %cast.rtn
} }
define amdgpu_ps void @global_dec_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps void @global_dec_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_dec_saddr_i32_nortn: ; GCN-LABEL: global_dec_saddr_i32_nortn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec v0, v1, s[2:3] ; GCN-NEXT: global_atomic_dec v0, v1, s[2:3]
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_dec_saddr_i32_nortn: ; GFX11-LABEL: global_dec_saddr_i32_nortn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[2:3] ; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[2:3]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%unused = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep0, i32 %data, i32 0, i32 0, i1 false) %unused = atomicrmw udec_wrap ptr addrspace(1) %gep0, i32 %data monotonic
ret void ret void
} }
define amdgpu_ps void @global_dec_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) { define amdgpu_ps void @global_dec_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
; GCN-LABEL: global_dec_saddr_i32_nortn_neg128: ; GCN-LABEL: global_dec_saddr_i32_nortn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec v0, v1, s[2:3] offset:-128 ; GCN-NEXT: global_atomic_dec v0, v1, s[2:3] offset:-128
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_dec_saddr_i32_nortn_neg128: ; GFX11-LABEL: global_dec_saddr_i32_nortn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[2:3] offset:-128
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%unused = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep1, i32 %data, i32 0, i32 0, i1 false) %unused = atomicrmw udec_wrap ptr addrspace(1) %gep1, i32 %data monotonic
ret void ret void
} }
define amdgpu_ps <2 x float> @global_dec_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps <2 x float> @global_dec_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_dec_saddr_i64_rtn: ; GCN-LABEL: global_dec_saddr_i64_rtn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GCN-NEXT: global_atomic_dec_x2 v[0:1], v0, v[1:2], s[2:3] glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_dec_saddr_i64_rtn: ; GFX11-LABEL: global_dec_saddr_i64_rtn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v0, v[1:2], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%rtn = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep0, i64 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep0, i64 %data monotonic
%cast.rtn = bitcast i64 %rtn to <2 x float> %cast.rtn = bitcast i64 %rtn to <2 x float>
ret <2 x float> %cast.rtn ret <2 x float> %cast.rtn
} }
define amdgpu_ps <2 x float> @global_dec_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps <2 x float> @global_dec_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_dec_saddr_i64_rtn_neg128: ; GCN-LABEL: global_dec_saddr_i64_rtn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GCN-NEXT: global_atomic_dec_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; return to shader part epilog ; GCN-NEXT: ; return to shader part epilog
; ;
; GFX11-LABEL: global_dec_saddr_i64_rtn_neg128: ; GFX11-LABEL: global_dec_saddr_i64_rtn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog ; GFX11-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%rtn = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep1, i64 %data, i32 0, i32 0, i1 false) %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep1, i64 %data monotonic
%cast.rtn = bitcast i64 %rtn to <2 x float> %cast.rtn = bitcast i64 %rtn to <2 x float>
ret <2 x float> %cast.rtn ret <2 x float> %cast.rtn
} }
define amdgpu_ps void @global_dec_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps void @global_dec_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_dec_saddr_i64_nortn: ; GCN-LABEL: global_dec_saddr_i64_nortn:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec_x2 v0, v[1:2], s[2:3] ; GCN-NEXT: global_atomic_dec_x2 v0, v[1:2], s[2:3]
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_dec_saddr_i64_nortn: ; GFX11-LABEL: global_dec_saddr_i64_nortn:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u64 v0, v[1:2], s[2:3] ; GFX11-NEXT: global_atomic_dec_u64 v0, v[1:2], s[2:3]
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%unused = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep0, i64 %data, i32 0, i32 0, i1 false) %unused = atomicrmw udec_wrap ptr addrspace(1) %gep0, i64 %data monotonic
ret void ret void
} }
define amdgpu_ps void @global_dec_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) { define amdgpu_ps void @global_dec_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
; GCN-LABEL: global_dec_saddr_i64_nortn_neg128: ; GCN-LABEL: global_dec_saddr_i64_nortn_neg128:
; GCN: ; %bb.0: ; GCN: ; %bb.0:
; GCN-NEXT: global_atomic_dec_x2 v0, v[1:2], s[2:3] offset:-128 ; GCN-NEXT: global_atomic_dec_x2 v0, v[1:2], s[2:3] offset:-128
; GCN-NEXT: s_endpgm ; GCN-NEXT: s_endpgm
; ;
; GFX11-LABEL: global_dec_saddr_i64_nortn_neg128: ; GFX11-LABEL: global_dec_saddr_i64_nortn_neg128:
; GFX11: ; %bb.0: ; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_dec_u64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: global_atomic_dec_u64 v0, v[1:2], s[2:3] offset:-128
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm ; GFX11-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64 %zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128 %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
%unused = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep1, i64 %data, i32 0, i32 0, i1 false) %unused = atomicrmw udec_wrap ptr addrspace(1) %gep1, i64 %data monotonic
ret void ret void
} }
attributes #0 = { argmemonly nounwind willreturn } attributes #0 = { argmemonly nounwind willreturn }

llvm/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll

Show All 10 Lines
; CHECK-LABEL: @objectsize_global_to_flat_i64( ; CHECK-LABEL: @objectsize_global_to_flat_i64(
; CHECK: %val = call i64 @llvm.objectsize.i64.p3(ptr addrspace(3) %global.ptr, i1 true, i1 false, i1 false) ; CHECK: %val = call i64 @llvm.objectsize.i64.p3(ptr addrspace(3) %global.ptr, i1 true, i1 false, i1 false)
define i64 @objectsize_global_to_flat_i64(ptr addrspace(3) %global.ptr) #0 { define i64 @objectsize_global_to_flat_i64(ptr addrspace(3) %global.ptr) #0 {
%cast = addrspacecast ptr addrspace(3) %global.ptr to ptr %cast = addrspacecast ptr addrspace(3) %global.ptr to ptr
%val = call i64 @llvm.objectsize.i64.p0(ptr %cast, i1 true, i1 false, i1 false) %val = call i64 @llvm.objectsize.i64.p0(ptr %cast, i1 true, i1 false, i1 false)
ret i64 %val ret i64 %val
} }
; CHECK-LABEL: @atomicinc_global_to_flat_i32(
; CHECK: call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %global.ptr, i32 %y, i32 0, i32 0, i1 false)
define i32 @atomicinc_global_to_flat_i32(ptr addrspace(1) %global.ptr, i32 %y) #0 {
%cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
%ret = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %cast, i32 %y, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK-LABEL: @atomicinc_group_to_flat_i32(
; CHECK: %ret = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %group.ptr, i32 %y, i32 0, i32 0, i1 false)
define i32 @atomicinc_group_to_flat_i32(ptr addrspace(3) %group.ptr, i32 %y) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %cast, i32 %y, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK-LABEL: @atomicinc_global_to_flat_i64(
; CHECK: call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %global.ptr, i64 %y, i32 0, i32 0, i1 false)
define i64 @atomicinc_global_to_flat_i64(ptr addrspace(1) %global.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 false)
ret i64 %ret
}
; CHECK-LABEL: @atomicinc_group_to_flat_i64(
; CHECK: call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %group.ptr, i64 %y, i32 0, i32 0, i1 false)
define i64 @atomicinc_group_to_flat_i64(ptr addrspace(3) %group.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 false)
ret i64 %ret
}
; CHECK-LABEL: @atomicdec_global_to_flat_i32(
; CHECK: call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %global.ptr, i32 %val, i32 0, i32 0, i1 false)
define i32 @atomicdec_global_to_flat_i32(ptr addrspace(1) %global.ptr, i32 %val) #0 {
%cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
%ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %cast, i32 %val, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK-LABEL: @atomicdec_group_to_flat_i32(
; CHECK: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %group.ptr, i32 %val, i32 0, i32 0, i1 false)
define i32 @atomicdec_group_to_flat_i32(ptr addrspace(3) %group.ptr, i32 %val) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %cast, i32 %val, i32 0, i32 0, i1 false)
ret i32 %ret
}
; CHECK-LABEL: @atomicdec_global_to_flat_i64(
; CHECK: call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %global.ptr, i64 %y, i32 0, i32 0, i1 false)
define i64 @atomicdec_global_to_flat_i64(ptr addrspace(1) %global.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 false)
ret i64 %ret
}
; CHECK-LABEL: @atomicdec_group_to_flat_i64(
; CHECK: call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %group.ptr, i64 %y, i32 0, i32 0, i1 false
define i64 @atomicdec_group_to_flat_i64(ptr addrspace(3) %group.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 false)
ret i64 %ret
}
; CHECK-LABEL: @volatile_atomicinc_group_to_flat_i64(
; CHECK-NEXT: %cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
; CHECK-NEXT: %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 true)
define i64 @volatile_atomicinc_group_to_flat_i64(ptr addrspace(3) %group.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 true)
ret i64 %ret
}
; CHECK-LABEL: @volatile_atomicdec_global_to_flat_i32(
; CHECK-NEXT: %cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
; CHECK-NEXT: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %cast, i32 %val, i32 0, i32 0, i1 true)
define i32 @volatile_atomicdec_global_to_flat_i32(ptr addrspace(1) %global.ptr, i32 %val) #0 {
%cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
%ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %cast, i32 %val, i32 0, i32 0, i1 true)
ret i32 %ret
}
; CHECK-LABEL: @volatile_atomicdec_group_to_flat_i32(
; CHECK-NEXT: %cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
; CHECK-NEXT: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %cast, i32 %val, i32 0, i32 0, i1 true)
define i32 @volatile_atomicdec_group_to_flat_i32(ptr addrspace(3) %group.ptr, i32 %val) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %cast, i32 %val, i32 0, i32 0, i1 true)
ret i32 %ret
}
; CHECK-LABEL: @volatile_atomicdec_global_to_flat_i64(
; CHECK-NEXT: %cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
; CHECK-NEXT: %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 true)
define i64 @volatile_atomicdec_global_to_flat_i64(ptr addrspace(1) %global.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(1) %global.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 true)
ret i64 %ret
}
; CHECK-LABEL: @volatile_atomicdec_group_to_flat_i64(
; CHECK-NEXT: %cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
; CHECK-NEXT: %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 true)
define i64 @volatile_atomicdec_group_to_flat_i64(ptr addrspace(3) %group.ptr, i64 %y) #0 {
%cast = addrspacecast ptr addrspace(3) %group.ptr to ptr
%ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %cast, i64 %y, i32 0, i32 0, i1 true)
ret i64 %ret
}
declare i32 @llvm.objectsize.i32.p0(ptr, i1, i1, i1) #1 declare i32 @llvm.objectsize.i32.p0(ptr, i1, i1, i1) #1
declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1) #1 declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1) #1
declare i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr nocapture, i32, i32, i32, i1) #2
declare i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr nocapture, i64, i32, i32, i1) #2
declare i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr nocapture, i32, i32, i32, i1) #2
declare i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr nocapture, i64, i32, i32, i1) #2
attributes #0 = { nounwind } attributes #0 = { nounwind }
attributes #1 = { nounwind readnone } attributes #1 = { nounwind readnone }
attributes #2 = { nounwind argmemonly } attributes #2 = { nounwind argmemonly }

llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll

Show First 20 Lines • Show All 81 Lines • ▼ Show 20 Lines.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader
%tmp7.0 = extractvalue { i32, i1 } %tmp7, 0 %tmp7.0 = extractvalue { i32, i1 } %tmp7, 0
%tmp8 = add nsw i32 %tmp7.0, %tmp4.0 %tmp8 = add nsw i32 %tmp7.0, %tmp4.0
atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
%indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
%exitcond = icmp eq i32 %indvars.iv.next, %n %exitcond = icmp eq i32 %indvars.iv.next, %n
br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
} }
; OPT-LABEL: @test_local_atomicinc_addressing_loop_uniform_index_max_offset_i32(
; OPT: .lr.ph.preheader:
; OPT: %scevgep2 = getelementptr i32, i32 addrspace(3)* %arg1, i32 16383
; OPT: br label %.lr.ph
; OPT: .lr.ph:
; OPT: %lsr.iv3 = phi i32 addrspace(3)* [ %scevgep4, %.lr.ph ], [ %scevgep2, %.lr.ph.preheader ]
; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ]
; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ]
; OPT: %tmp4 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %lsr.iv3, i32 undef, i32 0, i32 0, i1 false)
; OPT: %tmp7 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %lsr.iv1, i32 undef, i32 0, i32 0, i1 false)
; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv3, i32 1
define amdgpu_kernel void @test_local_atomicinc_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
bb:
%tmp = icmp sgt i32 %n, 0
br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
.lr.ph.preheader: ; preds = %bb
br label %.lr.ph
._crit_edge.loopexit: ; preds = %.lr.ph
br label %._crit_edge
._crit_edge: ; preds = %._crit_edge.loopexit, %bb
ret void
.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader
%indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
%tmp1 = add nuw nsw i32 %indvars.iv, 16383
%tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1
%tmp4 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %tmp3, i32 undef, i32 0, i32 0, i1 false)
%tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv
%tmp7 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %tmp6, i32 undef, i32 0, i32 0, i1 false)
%tmp8 = add nsw i32 %tmp7, %tmp4
atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
%indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
%exitcond = icmp eq i32 %indvars.iv.next, %n
br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
}
; OPT-LABEL: @test_local_atomicdec_addressing_loop_uniform_index_max_offset_i32(
; OPT: .lr.ph.preheader:
; OPT: %scevgep2 = getelementptr i32, i32 addrspace(3)* %arg1, i32 16383
; OPT: br label %.lr.ph
; OPT: .lr.ph:
; OPT: %lsr.iv3 = phi i32 addrspace(3)* [ %scevgep4, %.lr.ph ], [ %scevgep2, %.lr.ph.preheader ]
; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ]
; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ]
; OPT: %tmp4 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %lsr.iv3, i32 undef, i32 0, i32 0, i1 false)
; OPT: %tmp7 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %lsr.iv1, i32 undef, i32 0, i32 0, i1 false)
; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv3, i32 1
define amdgpu_kernel void @test_local_atomicdec_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
bb:
%tmp = icmp sgt i32 %n, 0
br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
.lr.ph.preheader: ; preds = %bb
br label %.lr.ph
._crit_edge.loopexit: ; preds = %.lr.ph
br label %._crit_edge
._crit_edge: ; preds = %._crit_edge.loopexit, %bb
ret void
.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader
%indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
%tmp1 = add nuw nsw i32 %indvars.iv, 16383
%tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1
%tmp4 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %tmp3, i32 undef, i32 0, i32 0, i1 false)
%tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv
%tmp7 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %tmp6, i32 undef, i32 0, i32 0, i1 false)
%tmp8 = add nsw i32 %tmp7, %tmp4
atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
%indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
%exitcond = icmp eq i32 %indvars.iv.next, %n
br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
}
declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #1
declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #1
attributes #0 = { nounwind } attributes #0 = { nounwind }
attributes #1 = { nounwind argmemonly } attributes #1 = { nounwind argmemonly }

llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll

Show First 20 Lines • Show All 136 Lines • ▼ Show 20 Lines
define i64 @invalid_nonconstant_fcmp_code(float %a, float %b, i32 %c) { define i64 @invalid_nonconstant_fcmp_code(float %a, float %b, i32 %c) {
; CHECK: immarg operand has non-immediate parameter ; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %c ; CHECK-NEXT: i32 %c
; CHECK-NEXT: %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 %c) ; CHECK-NEXT: %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 %c)
%result = call i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 %c) %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float %a, float %b, i32 %c)
ret i64 %result ret i64 %result
} }
declare i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1)
define amdgpu_kernel void @invalid_atomic_inc(ptr addrspace(1) %out, ptr addrspace(3) %ptr, i32 %var, i1 %bool) {
; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %var
; CHECK-NEXT: %result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 %var, i32 0, i1 false)
%result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 %var, i32 0, i1 false)
; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %var
; CHECK-NEXT: %result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 %var, i1 false)
%result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 %var, i1 false)
; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i1 %bool
; CHECK-NEXT: %result2 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 %bool)
%result2 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 %bool)
ret void
}
declare i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1)
define amdgpu_kernel void @invalid_atomic_dec(ptr addrspace(1) %out, ptr addrspace(3) %ptr, i32 %var, i1 %bool) {
; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %var
; CHECK-NEXT: %result0 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 %var, i32 0, i1 false)
%result0 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 %var, i32 0, i1 false)
; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i32 %var
; CHECK-NEXT: %result1 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 %var, i1 false)
%result1 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 %var, i1 false)
; CHECK: immarg operand has non-immediate parameter
; CHECK-NEXT: i1 %bool
; CHECK-NEXT: %result2 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 %bool)
%result2 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 %bool)
ret void
}
declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1) declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1)
define amdgpu_kernel void @test_div_scale_f32_val_undef_undef(ptr addrspace(1) %out) { define amdgpu_kernel void @test_div_scale_f32_val_undef_undef(ptr addrspace(1) %out) {
; CHECK: immarg operand has non-immediate parameter ; CHECK: immarg operand has non-immediate parameter
; CHECK: i1 undef ; CHECK: i1 undef
; CHECK: %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.000000e+00, float undef, i1 undef) ; CHECK: %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.000000e+00, float undef, i1 undef)
%result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.0, float undef, i1 undef) %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.0, float undef, i1 undef)
%result0 = extractvalue { float, i1 } %result, 0 %result0 = extractvalue { float, i1 } %result, 0
store float %result0, ptr addrspace(1) %out, align 4 store float %result0, ptr addrspace(1) %out, align 4
▲ Show 20 Lines • Show All 524 Lines • Show Last 20 Lines
llvm/test/Bitcode/amdgcn-atomic.ll