This is an archive of the discontinued LLVM Phabricator instance.

Differential D99902

Copy syncscope when expanding atomicrmw into cmpxchg loop
ClosedPublic

Authored by rampitec on Apr 5 2021, 3:03 PM.

Details

Reviewers
kzhuravl
arsenm
Commits
rG30b3aab3299a: Copy syncscope when expanding atomicrmw into cmpxchg loop
Summary

Fixes: SWDEV-280070

Diff Detail

Event Timeline

rampitec created this revision.Apr 5 2021, 3:03 PM
Herald added subscribers: kerbowa, jfb, hiraditya and 2 others. · View Herald TranscriptApr 5 2021, 3:03 PM
rampitec requested review of this revision.Apr 5 2021, 3:03 PM
Herald added a project: Restricted Project. · View Herald TranscriptApr 5 2021, 3:03 PM
Herald added a subscriber: wdng. · View Herald Transcript
Harbormaster completed remote builds in B97180: Diff 335336.Apr 5 2021, 3:57 PM
arsenm accepted this revision.Apr 5 2021, 4:48 PM
This revision is now accepted and ready to land.Apr 5 2021, 4:48 PM
rampitec updated this revision to Diff 335358.Apr 5 2021, 5:20 PM

Fixed typo in test name.

arsenm accepted this revision.Apr 5 2021, 5:29 PM
This revision was landed with ongoing or failed builds.Apr 5 2021, 5:30 PM
Closed by commit rG30b3aab3299a: Copy syncscope when expanding atomicrmw into cmpxchg loop (authored by rampitec). · Explain Why
This revision was automatically updated to reflect the committed changes.
rampitec added a commit: rG30b3aab3299a: Copy syncscope when expanding atomicrmw into cmpxchg loop.
Harbormaster completed remote builds in B97198: Diff 335358.Apr 5 2021, 6:09 PM

Revision Contents

PathSize
llvm/
include/
llvm/
CodeGen/
2 lines
lib/
CodeGen/
22 lines
test/
CodeGen/
AMDGPU/
76 lines
19 lines
18 lines
Transforms/
AtomicExpand/
AMDGPU/
178 lines

Diff 335336

llvm/include/llvm/CodeGen/AtomicExpandUtils.h

Show All 17 Lines
class AtomicRMWInst; class AtomicRMWInst;
class Value; class Value;
/// Parameters (see the expansion example below): /// Parameters (see the expansion example below):
/// (the builder, %addr, %loaded, %new_val, ordering, /// (the builder, %addr, %loaded, %new_val, ordering,
/// /* OUT */ %success, /* OUT */ %new_loaded) /// /* OUT */ %success, /* OUT */ %new_loaded)
using CreateCmpXchgInstFun = using CreateCmpXchgInstFun =
function_ref<void(IRBuilder<> &, Value *, Value *, Value *, Align, function_ref<void(IRBuilder<> &, Value *, Value *, Value *, Align,
AtomicOrdering, Value *&, Value *&)>; AtomicOrdering, SyncScope::ID, Value *&, Value *&)>;
/// Expand an atomic RMW instruction into a loop utilizing /// Expand an atomic RMW instruction into a loop utilizing
/// cmpxchg. You'll want to make sure your target machine likes cmpxchg /// cmpxchg. You'll want to make sure your target machine likes cmpxchg
/// instructions in the first place and that there isn't another, better, /// instructions in the first place and that there isn't another, better,
/// transformation available (for example AArch32/AArch64 have linked loads). /// transformation available (for example AArch32/AArch64 have linked loads).
/// ///
/// This is useful in passes which can't rewrite the more exotic RMW /// This is useful in passes which can't rewrite the more exotic RMW
/// instructions directly into a platform specific intrinsics (because, say, /// instructions directly into a platform specific intrinsics (because, say,
Show All 30 Lines

llvm/lib/CodeGen/AtomicExpandPass.cpp

Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines private:
AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI); AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
bool expandPartwordCmpXchg(AtomicCmpXchgInst *I); bool expandPartwordCmpXchg(AtomicCmpXchgInst *I);
void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI); void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI); void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI); AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
static Value *insertRMWCmpXchgLoop( static Value *insertRMWCmpXchgLoop(
IRBuilder<> &Builder, Type *ResultType, Value *Addr, Align AddrAlign, IRBuilder<> &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder, AtomicOrdering MemOpOrder, SyncScope::ID SSID,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp, function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
CreateCmpXchgInstFun CreateCmpXchg); CreateCmpXchgInstFun CreateCmpXchg);
bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI); bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
bool isIdempotentRMW(AtomicRMWInst *RMWI); bool isIdempotentRMW(AtomicRMWInst *RMWI);
bool simplifyIdempotentRMW(AtomicRMWInst *RMWI); bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
▲ Show 20 Lines • Show All 353 Lines • ▼ Show 20 Linesbool AtomicExpand::expandAtomicStore(StoreInst *SI) {
SI->eraseFromParent(); SI->eraseFromParent();
// Now we have an appropriate swap instruction, lower it as usual. // Now we have an appropriate swap instruction, lower it as usual.
return tryExpandAtomicRMW(AI); return tryExpandAtomicRMW(AI);
} }
static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
Value *Loaded, Value *NewVal, Align AddrAlign, Value *Loaded, Value *NewVal, Align AddrAlign,
AtomicOrdering MemOpOrder, Value *&Success, AtomicOrdering MemOpOrder, SyncScope::ID SSID,
Value *&NewLoaded) { Value *&Success, Value *&NewLoaded) {
Type *OrigTy = NewVal->getType(); Type *OrigTy = NewVal->getType();
// This code can go away when cmpxchg supports FP types. // This code can go away when cmpxchg supports FP types.
bool NeedBitcast = OrigTy->isFloatingPointTy(); bool NeedBitcast = OrigTy->isFloatingPointTy();
if (NeedBitcast) { if (NeedBitcast) {
IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits()); IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
unsigned AS = Addr->getType()->getPointerAddressSpace(); unsigned AS = Addr->getType()->getPointerAddressSpace();
Addr = Builder.CreateBitCast(Addr, IntTy->getPointerTo(AS)); Addr = Builder.CreateBitCast(Addr, IntTy->getPointerTo(AS));
NewVal = Builder.CreateBitCast(NewVal, IntTy); NewVal = Builder.CreateBitCast(NewVal, IntTy);
Loaded = Builder.CreateBitCast(Loaded, IntTy); Loaded = Builder.CreateBitCast(Loaded, IntTy);
} }
Value *Pair = Builder.CreateAtomicCmpXchg( Value *Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, AddrAlign, MemOpOrder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder), SSID);
Success = Builder.CreateExtractValue(Pair, 1, "success"); Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
if (NeedBitcast) if (NeedBitcast)
NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy); NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
} }
/// Emit IR to implement the given atomicrmw operation on values in registers, /// Emit IR to implement the given atomicrmw operation on values in registers,
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines
/// ///
/// It will create an LL/SC or cmpxchg loop, as appropriate, the same /// It will create an LL/SC or cmpxchg loop, as appropriate, the same
/// way as a typical atomicrmw expansion. The only difference here is /// way as a typical atomicrmw expansion. The only difference here is
/// that the operation inside of the loop may operate upon only a /// that the operation inside of the loop may operate upon only a
/// part of the value. /// part of the value.
void AtomicExpand::expandPartwordAtomicRMW( void AtomicExpand::expandPartwordAtomicRMW(
AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) { AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
AtomicOrdering MemOpOrder = AI->getOrdering(); AtomicOrdering MemOpOrder = AI->getOrdering();
SyncScope::ID SSID = AI->getSyncScopeID();
IRBuilder<> Builder(AI); IRBuilder<> Builder(AI);
PartwordMaskValues PMV = PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted = Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
PMV.ShiftAmt, "ValOperand_Shifted"); PMV.ShiftAmt, "ValOperand_Shifted");
auto PerformPartwordOp = [&](IRBuilder<> &Builder, Value *Loaded) { auto PerformPartwordOp = [&](IRBuilder<> &Builder, Value *Loaded) {
return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded, return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded,
ValOperand_Shifted, AI->getValOperand(), PMV); ValOperand_Shifted, AI->getValOperand(), PMV);
}; };
Value *OldResult; Value *OldResult;
if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) { if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) {
OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
PMV.AlignedAddrAlignment, MemOpOrder, PMV.AlignedAddrAlignment, MemOpOrder,
Lint: Pre-merge checks
Inline Actions

clang-format: please reformat the code

-                                     PMV.AlignedAddrAlignment, MemOpOrder,
-                                     SSID, PerformPartwordOp,
-                                     createCmpXchgInstFun);
+                                     PMV.AlignedAddrAlignment, MemOpOrder, SSID,
+                                     PerformPartwordOp, createCmpXchgInstFun);
Lint: Pre-merge checks: clang-format: please reformat the code ``` - PMV.
PerformPartwordOp, createCmpXchgInstFun); SSID, PerformPartwordOp,
createCmpXchgInstFun);
} else { } else {
assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC); assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC);
OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
PMV.AlignedAddrAlignment, MemOpOrder, PMV.AlignedAddrAlignment, MemOpOrder,
PerformPartwordOp); PerformPartwordOp);
} }
Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
▲ Show 20 Lines • Show All 587 Lines • ▼ Show 20 Lines if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {
tryExpandAtomicLoad(ResultingLoad); tryExpandAtomicLoad(ResultingLoad);
return true; return true;
} }
return false; return false;
} }
Value *AtomicExpand::insertRMWCmpXchgLoop( Value *AtomicExpand::insertRMWCmpXchgLoop(
IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign, IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
AtomicOrdering MemOpOrder, AtomicOrdering MemOpOrder, SyncScope::ID SSID,
function_ref<Value *(IRBuilder<> &, Value *)> PerformOp, function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
CreateCmpXchgInstFun CreateCmpXchg) { CreateCmpXchgInstFun CreateCmpXchg) {
LLVMContext &Ctx = Builder.getContext(); LLVMContext &Ctx = Builder.getContext();
BasicBlock *BB = Builder.GetInsertBlock(); BasicBlock *BB = Builder.GetInsertBlock();
Function *F = BB->getParent(); Function *F = BB->getParent();
// Given: atomicrmw some_op iN* %addr, iN %incr ordering // Given: atomicrmw some_op iN* %addr, iN %incr ordering
// //
Show All 31 LinesValue *AtomicExpand::insertRMWCmpXchgLoop(
Value *NewLoaded = nullptr; Value *NewLoaded = nullptr;
Value *Success = nullptr; Value *Success = nullptr;
CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign, CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
MemOpOrder == AtomicOrdering::Unordered MemOpOrder == AtomicOrdering::Unordered
? AtomicOrdering::Monotonic ? AtomicOrdering::Monotonic
: MemOpOrder, : MemOpOrder,
Success, NewLoaded); SSID, Success, NewLoaded);
assert(Success && NewLoaded); assert(Success && NewLoaded);
Loaded->addIncoming(NewLoaded, LoopBB); Loaded->addIncoming(NewLoaded, LoopBB);
Builder.CreateCondBr(Success, ExitBB, LoopBB); Builder.CreateCondBr(Success, ExitBB, LoopBB);
Builder.SetInsertPoint(ExitBB, ExitBB->begin()); Builder.SetInsertPoint(ExitBB, ExitBB->begin());
return NewLoaded; return NewLoaded;
Show All 20 Lines
} }
// Note: This function is exposed externally by AtomicExpandUtils.h // Note: This function is exposed externally by AtomicExpandUtils.h
bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
CreateCmpXchgInstFun CreateCmpXchg) { CreateCmpXchgInstFun CreateCmpXchg) {
IRBuilder<> Builder(AI); IRBuilder<> Builder(AI);
Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop( Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(), Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
AI->getOrdering(), AI->getOrdering(), AI->getSyncScopeID(),
[&](IRBuilder<> &Builder, Value *Loaded) { [&](IRBuilder<> &Builder, Value *Loaded) {
return performAtomicOp(AI->getOperation(), Builder, Loaded, return performAtomicOp(AI->getOperation(), Builder, Loaded,
AI->getValOperand()); AI->getValOperand());
}, },
CreateCmpXchg); CreateCmpXchg);
AI->replaceAllUsesWith(Loaded); AI->replaceAllUsesWith(Loaded);
AI->eraseFromParent(); AI->eraseFromParent();
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Linesvoid AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
// The expansion failed: either there were no libcalls at all for // The expansion failed: either there were no libcalls at all for
// the operation (min/max), or there were only size-specialized // the operation (min/max), or there were only size-specialized
// libcalls (add/sub/etc) and we needed a generic. So, expand to a // libcalls (add/sub/etc) and we needed a generic. So, expand to a
// CAS libcall, via a CAS loop, instead. // CAS libcall, via a CAS loop, instead.
if (!Success) { if (!Success) {
expandAtomicRMWToCmpXchg( expandAtomicRMWToCmpXchg(
I, [this](IRBuilder<> &Builder, Value *Addr, Value *Loaded, I, [this](IRBuilder<> &Builder, Value *Addr, Value *Loaded,
Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder, Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
Value *&Success, Value *&NewLoaded) { SyncScope::ID SSID, Value *&Success, Value *&NewLoaded) {
// Create the CAS instruction normally... // Create the CAS instruction normally...
AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg( AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, Alignment, MemOpOrder, Addr, Loaded, NewVal, Alignment, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder), SSID);
Success = Builder.CreateExtractValue(Pair, 1, "success"); Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
// ...and then expand the CAS into a libcall. // ...and then expand the CAS into a libcall.
expandAtomicCASToLibcall(Pair); expandAtomicCASToLibcall(Pair);
}); });
} }
} }
▲ Show 20 Lines • Show All 212 Lines • Show Last 20 Lines

llvm/test/CodeGen/AMDGPU/expand-atomicrmw-sychscope.ll

  • This file was added.
; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
; Check that syncscope it copied from atomicrmw to cmpxchg during expansion.
; There should be no scc unless we have system scope.
; GCN-LABEL: {{^}}expand_atomicrmw_agent:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_agent(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("agent") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_workgroup:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_workgroup(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("workgroup") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_wavefront:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_wavefront(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("wavefront") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_agent_one_as:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_agent_one_as(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("agent-one-as") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_workgroup_one_as:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_workgroup_one_as(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("workgroup-one-as") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_wavefront_one_as:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_wavefront_one_as(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("wavefront-one-as") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_singlethread_one_as:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc{{$}}
define void @expand_atomicrmw_singlethread_one_as(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("singlethread-one-as") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_one_as:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc scc{{$}}
define void @expand_atomicrmw_one_as(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 syncscope("one-as") monotonic, align 4
ret void
}
; GCN-LABEL: {{^}}expand_atomicrmw_system:
; GCN: global_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9:]+}}], v[{{[0-9:]+}}], off glc scc{{$}}
define void @expand_atomicrmw_system(float addrspace(1)* nocapture %arg) {
entry:
%ret = atomicrmw fadd float addrspace(1)* %arg, float 1.000000e+00 monotonic, align 4
ret void
}

llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll

Show First 20 Lines • Show All 468 Lines • ▼ Show 20 Lines
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: BB26_1: ; %atomicrmw.start ; GFX90A-NEXT: BB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 ; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc scc ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc scc
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
Show All 14 Lines
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: BB27_1: ; %atomicrmw.start ; GFX90A-NEXT: BB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: v_mov_b32_e32 v4, 0
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 ; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc scc ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_execnz BB27_1 ; GFX90A-NEXT: s_cbranch_execnz BB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
; GFX90A-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX90A-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: BB31_1: ; %atomicrmw.start ; GFX90A-NEXT: BB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], 4.0 ; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], 4.0
; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc scc ; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc scc
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1] ; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: BB36_1: ; %atomicrmw.start ; GFX90A-NEXT: BB36_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 ; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc scc ; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc scc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_execnz BB36_1 ; GFX90A-NEXT: s_cbranch_execnz BB36_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm ; GFX90A-NEXT: s_endpgm
main_body: main_body:
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1] ; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0 ; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: BB39_1: ; %atomicrmw.start ; GFX90A-NEXT: BB39_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], 4.0 ; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], 4.0
; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc scc ; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc scc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5] ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz BB39_1 ; GFX90A-NEXT: s_cbranch_execnz BB39_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v2 ; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: v_mov_b32_e32 v1, v3 ; GFX90A-NEXT: v_mov_b32_e32 v1, v3
▲ Show 20 Lines • Show All 181 Lines • Show Last 20 Lines

llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll

Show First 20 Lines • Show All 178 Lines • ▼ Show 20 Lines
; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, s4 ; GFX90A-NEXT: v_mov_b32_e32 v0, s4
; GFX90A-NEXT: BB1_1: ; %atomicrmw.start ; GFX90A-NEXT: BB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v1, v0 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1 ; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc scc ; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_execnz BB1_1 ; GFX90A-NEXT: s_cbranch_execnz BB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s4 ; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: BB3_1: ; %atomicrmw.start ; GFX90A-NEXT: BB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1 ; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc scc ; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_execnz BB3_1 ; GFX90A-NEXT: s_cbranch_execnz BB3_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0) ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v0, s4 ; GFX900-NEXT: v_mov_b32_e32 v0, s4
; GFX900-NEXT: BB5_1: ; %atomicrmw.start ; GFX900-NEXT: BB5_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v1, v0 ; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: v_mov_b32_e32 v2, 0 ; GFX900-NEXT: v_mov_b32_e32 v2, 0
; GFX900-NEXT: v_add_f32_e32 v0, 4.0, v1 ; GFX900-NEXT: v_add_f32_e32 v0, 4.0, v1
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc ; GFX900-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0) ; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol ; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz BB5_1 ; GFX900-NEXT: s_cbranch_execnz BB5_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
Show All 9 Lines
; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0 ; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v0, s4 ; GFX908-NEXT: v_mov_b32_e32 v0, s4
; GFX908-NEXT: BB5_1: ; %atomicrmw.start ; GFX908-NEXT: BB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v1, v0 ; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v2, 0 ; GFX908-NEXT: v_mov_b32_e32 v2, 0
; GFX908-NEXT: v_add_f32_e32 v0, 4.0, v1 ; GFX908-NEXT: v_add_f32_e32 v0, 4.0, v1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc ; GFX908-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol ; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX908-NEXT: s_cbranch_execnz BB5_1 ; GFX908-NEXT: s_cbranch_execnz BB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
Show All 10 Lines
; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, s4 ; GFX90A-NEXT: v_mov_b32_e32 v0, s4
; GFX90A-NEXT: BB5_1: ; %atomicrmw.start ; GFX90A-NEXT: BB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v1, v0 ; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1 ; GFX90A-NEXT: v_add_f32_e32 v0, 4.0, v1
; GFX90A-NEXT: buffer_wbl2 ; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc scc ; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc scc
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
Show All 11 Lines
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: s_mov_b32 s2, 0 ; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: BB5_1: ; %atomicrmw.start ; GFX10-NEXT: BB5_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_add_f32_e32 v0, 4.0, v1 ; GFX10-NEXT: v_add_f32_e32 v0, 4.0, v1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc ; GFX10-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv ; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
▲ Show 20 Lines • Show All 59 Lines • Show Last 20 Lines

llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -atomic-expand %s | FileCheck -check-prefix=GFX908 %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -atomic-expand %s | FileCheck -check-prefix=GFX908 %s
define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) { define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_flat( ; CI-LABEL: @test_atomicrmw_fadd_f32_flat(
; CI-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4 ; CI-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32* ; CI-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret float [[TMP6]] ; CI-NEXT: ret float [[TMP6]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_flat( ; GFX9-LABEL: @test_atomicrmw_fadd_f32_flat(
; GFX9-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4 ; GFX9-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret float [[TMP6]] ; GFX9-NEXT: ret float [[TMP6]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_flat( ; GFX908-LABEL: @test_atomicrmw_fadd_f32_flat(
; GFX908-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4 ; GFX908-NEXT: [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start: ; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32* ; GFX908-NEXT: [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end: ; GFX908: atomicrmw.end:
; GFX908-NEXT: ret float [[TMP6]] ; GFX908-NEXT: ret float [[TMP6]]
; ;
%res = atomicrmw fadd float* %ptr, float %value seq_cst %res = atomicrmw fadd float* %ptr, float %value seq_cst
ret float %res ret float %res
} }
define float @test_atomicrmw_fadd_f32_global(float addrspace(1)* %ptr, float %value) { define float @test_atomicrmw_fadd_f32_global(float addrspace(1)* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_global( ; CI-LABEL: @test_atomicrmw_fadd_f32_global(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret float [[TMP6]] ; CI-NEXT: ret float [[TMP6]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_global( ; GFX9-LABEL: @test_atomicrmw_fadd_f32_global(
; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret float [[TMP6]] ; GFX9-NEXT: ret float [[TMP6]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_global( ; GFX908-LABEL: @test_atomicrmw_fadd_f32_global(
; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start: ; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end: ; GFX908: atomicrmw.end:
; GFX908-NEXT: ret float [[TMP6]] ; GFX908-NEXT: ret float [[TMP6]]
; ;
%res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
ret float %res ret float %res
} }
define void @test_atomicrmw_fadd_f32_global_no_use_ieee(float addrspace(1)* %ptr, float %value) { define void @test_atomicrmw_fadd_f32_global_no_use_ieee(float addrspace(1)* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee( ; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret void ; CI-NEXT: ret void
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee( ; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee(
; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret void ; GFX9-NEXT: ret void
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee( ; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee(
; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start: ; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end: ; GFX908: atomicrmw.end:
; GFX908-NEXT: ret void ; GFX908-NEXT: ret void
; ;
%res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
ret void ret void
} }
define void @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(float addrspace(1)* %ptr, float %value) #0 { define void @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(float addrspace(1)* %ptr, float %value) #0 {
; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush( ; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret void ; CI-NEXT: ret void
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush( ; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(
; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4 ; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret void ; GFX9-NEXT: ret void
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush( ; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(
; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(1)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst ; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(1)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4
; GFX908-NEXT: ret void ; GFX908-NEXT: ret void
; ;
%res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
ret void ret void
} }
define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) { define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_local( ; CI-LABEL: @test_atomicrmw_fadd_f32_local(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4 ; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(3)* [[PTR]] to i32 addrspace(3)* ; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(3)* [[PTR]] to i32 addrspace(3)*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32 ; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32 ; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(3)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(3)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float ; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret float [[TMP6]] ; CI-NEXT: ret float [[TMP6]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_local( ; GFX9-LABEL: @test_atomicrmw_fadd_f32_local(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4
; GFX9-NEXT: ret float [[RES]] ; GFX9-NEXT: ret float [[RES]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_local( ; GFX908-LABEL: @test_atomicrmw_fadd_f32_local(
; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst ; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4
; GFX908-NEXT: ret float [[RES]] ; GFX908-NEXT: ret float [[RES]]
; ;
%res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst %res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst
ret float %res ret float %res
} }
define half @test_atomicrmw_fadd_f16_flat(half* %ptr, half %value) { define half @test_atomicrmw_fadd_f16_flat(half* %ptr, half %value) {
; CI-LABEL: @test_atomicrmw_fadd_f16_flat( ; CI-LABEL: @test_atomicrmw_fadd_f16_flat(
; CI-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; CI-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; CI-NEXT: ret half [[RES]] ; CI-NEXT: ret half [[RES]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat( ; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; GFX9-NEXT: ret half [[RES]] ; GFX9-NEXT: ret half [[RES]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f16_flat( ; GFX908-LABEL: @test_atomicrmw_fadd_f16_flat(
; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; GFX908-NEXT: ret half [[RES]] ; GFX908-NEXT: ret half [[RES]]
; ;
%res = atomicrmw fadd half* %ptr, half %value seq_cst %res = atomicrmw fadd half* %ptr, half %value seq_cst
ret half %res ret half %res
} }
define half @test_atomicrmw_fadd_f16_global(half addrspace(1)* %ptr, half %value) { define half @test_atomicrmw_fadd_f16_global(half addrspace(1)* %ptr, half %value) {
; CI-LABEL: @test_atomicrmw_fadd_f16_global( ; CI-LABEL: @test_atomicrmw_fadd_f16_global(
; CI-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; CI-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; CI-NEXT: ret half [[RES]] ; CI-NEXT: ret half [[RES]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f16_global( ; GFX9-LABEL: @test_atomicrmw_fadd_f16_global(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; GFX9-NEXT: ret half [[RES]] ; GFX9-NEXT: ret half [[RES]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f16_global( ; GFX908-LABEL: @test_atomicrmw_fadd_f16_global(
; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; GFX908-NEXT: ret half [[RES]] ; GFX908-NEXT: ret half [[RES]]
; ;
%res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst %res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst
ret half %res ret half %res
} }
define half @test_atomicrmw_fadd_f16_local(half addrspace(3)* %ptr, half %value) { define half @test_atomicrmw_fadd_f16_local(half addrspace(3)* %ptr, half %value) {
; CI-LABEL: @test_atomicrmw_fadd_f16_local( ; CI-LABEL: @test_atomicrmw_fadd_f16_local(
; CI-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; CI-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; CI-NEXT: ret half [[RES]] ; CI-NEXT: ret half [[RES]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f16_local( ; GFX9-LABEL: @test_atomicrmw_fadd_f16_local(
; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; GFX9-NEXT: ret half [[RES]] ; GFX9-NEXT: ret half [[RES]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f16_local( ; GFX908-LABEL: @test_atomicrmw_fadd_f16_local(
; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst ; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst, align 2
; GFX908-NEXT: ret half [[RES]] ; GFX908-NEXT: ret half [[RES]]
; ;
%res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst %res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst
ret half %res ret half %res
} }
define double @test_atomicrmw_fadd_f64_flat(double* %ptr, double %value) { define double @test_atomicrmw_fadd_f64_flat(double* %ptr, double %value) {
; CI-LABEL: @test_atomicrmw_fadd_f64_flat( ; CI-LABEL: @test_atomicrmw_fadd_f64_flat(
; CI-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8 ; CI-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64* ; CI-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret double [[TMP6]] ; CI-NEXT: ret double [[TMP6]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat( ; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat(
; GFX9-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8 ; GFX9-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret double [[TMP6]] ; GFX9-NEXT: ret double [[TMP6]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat( ; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat(
; GFX908-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8 ; GFX908-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start: ; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64* ; GFX908-NEXT: [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end: ; GFX908: atomicrmw.end:
; GFX908-NEXT: ret double [[TMP6]] ; GFX908-NEXT: ret double [[TMP6]]
; ;
%res = atomicrmw fadd double* %ptr, double %value seq_cst %res = atomicrmw fadd double* %ptr, double %value seq_cst
ret double %res ret double %res
} }
define double @test_atomicrmw_fadd_f64_global(double addrspace(1)* %ptr, double %value) { define double @test_atomicrmw_fadd_f64_global(double addrspace(1)* %ptr, double %value) {
; CI-LABEL: @test_atomicrmw_fadd_f64_global( ; CI-LABEL: @test_atomicrmw_fadd_f64_global(
; CI-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8 ; CI-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)* ; CI-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret double [[TMP6]] ; CI-NEXT: ret double [[TMP6]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f64_global( ; GFX9-LABEL: @test_atomicrmw_fadd_f64_global(
; GFX9-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8 ; GFX9-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret double [[TMP6]] ; GFX9-NEXT: ret double [[TMP6]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f64_global( ; GFX908-LABEL: @test_atomicrmw_fadd_f64_global(
; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8 ; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start: ; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)* ; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end: ; GFX908: atomicrmw.end:
; GFX908-NEXT: ret double [[TMP6]] ; GFX908-NEXT: ret double [[TMP6]]
; ;
%res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst %res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst
ret double %res ret double %res
} }
define double @test_atomicrmw_fadd_f64_local(double addrspace(3)* %ptr, double %value) { define double @test_atomicrmw_fadd_f64_local(double addrspace(3)* %ptr, double %value) {
; CI-LABEL: @test_atomicrmw_fadd_f64_local( ; CI-LABEL: @test_atomicrmw_fadd_f64_local(
; CI-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8 ; CI-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
; CI-NEXT: br label [[ATOMICRMW_START:%.*]] ; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start: ; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)* ; CI-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; CI-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end: ; CI: atomicrmw.end:
; CI-NEXT: ret double [[TMP6]] ; CI-NEXT: ret double [[TMP6]]
; ;
; GFX9-LABEL: @test_atomicrmw_fadd_f64_local( ; GFX9-LABEL: @test_atomicrmw_fadd_f64_local(
; GFX9-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8 ; GFX9-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start: ; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)* ; GFX9-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end: ; GFX9: atomicrmw.end:
; GFX9-NEXT: ret double [[TMP6]] ; GFX9-NEXT: ret double [[TMP6]]
; ;
; GFX908-LABEL: @test_atomicrmw_fadd_f64_local( ; GFX908-LABEL: @test_atomicrmw_fadd_f64_local(
; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8 ; GFX908-NEXT: [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] ; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start: ; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ] ; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]] ; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)* ; GFX908-NEXT: [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64 ; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW]] to i64
; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64 ; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst ; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1 ; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0 ; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double ; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] ; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end: ; GFX908: atomicrmw.end:
; GFX908-NEXT: ret double [[TMP6]] ; GFX908-NEXT: ret double [[TMP6]]
; ;
%res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst %res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst
ret double %res ret double %res
} }
define float @test_atomicrmw_fadd_f32_global_agent(float addrspace(1)* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_global_agent(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") monotonic monotonic, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end:
; CI-NEXT: ret float [[TMP6]]
;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_agent(
; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") monotonic monotonic, align 4
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret float [[TMP6]]
;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_agent(
; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") monotonic monotonic, align 4
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end:
; GFX908-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fadd float addrspace(1)* %ptr, float %value syncscope("agent") monotonic
ret float %res
}
define float @test_atomicrmw_fadd_f32_global_one_as(float addrspace(1)* %ptr, float %value) {
; CI-LABEL: @test_atomicrmw_fadd_f32_global_one_as(
; CI-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] syncscope("one-as") monotonic monotonic, align 4
; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; CI-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CI: atomicrmw.end:
; CI-NEXT: ret float [[TMP6]]
;
; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_one_as(
; GFX9-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] syncscope("one-as") monotonic monotonic, align 4
; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX9-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret float [[TMP6]]
;
; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_one_as(
; GFX908-NEXT: [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[NEW]] to i32
; GFX908-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] syncscope("one-as") monotonic monotonic, align 4
; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
; GFX908-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GFX908: atomicrmw.end:
; GFX908-NEXT: ret float [[TMP6]]
;
%res = atomicrmw fadd float addrspace(1)* %ptr, float %value syncscope("one-as") monotonic
ret float %res
}
attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }