This is an archive of the discontinued LLVM Phabricator instance.

Differential D103284

[AArch64][RISCV] Make sure isel correctly honors failure orderings.
ClosedPublic

Authored by efriedma on May 27 2021, 1:52 PM.

Details

Reviewers
craig.topper
jyknight
t.p.northover
Commits
rG0b3b0a727ad6: [AArch64][RISCV] Make sure isel correctly honors failure orderings.
Summary

If a cmpxchg specifies acquire or seq_cst on failure, make sure we generate code consistent with that ordering even if the success ordering is not acquire/seq_cst.

At one point, it was ambiguous whether this sort of construct was valid, but the C++ standad and LLVM now accept arbitrary combinations of success/failure orderings.

This doesn't address the corresponding issue in AtomicExpand. (This was reported as https://bugs.llvm.org/show_bug.cgi?id=33332 .)

Fixes https://bugs.llvm.org/show_bug.cgi?id=50512.

Diff Detail

Event Timeline

efriedma created this revision.May 27 2021, 1:52 PM
Herald added subscribers: vkmr, frasercrmck, danielkiss and 28 others. · View Herald TranscriptMay 27 2021, 1:52 PM
efriedma requested review of this revision.May 27 2021, 1:52 PM
Herald added a project: Restricted Project. · View Herald TranscriptMay 27 2021, 1:52 PM
Herald added a subscriber: MaskRay. · View Herald Transcript
Harbormaster completed remote builds in B106598: Diff 348377.May 27 2021, 2:54 PM
t.p.northover accepted this revision.May 28 2021, 3:05 AM

LGTM.

This revision is now accepted and ready to land.May 28 2021, 3:05 AM
jyknight added inline comments.May 28 2021, 5:51 AM
llvm/include/llvm/CodeGen/MachineMemOperand.h
261

I think this is a confusing API now. The reason it made sense to have getOrdering return the success ordering for a cmpxchg before is because that was always stronger than the failure ordering. So, using the success order was a safe "default" ordering. That's no longer the case.

Maybe we should have getOrdering should return the merged ordering, and rename the existing getOrdering to getSuccessOrdering.

Or, actually I like this better: remove getOrdering entirely, by renaming to getSuccessOrdering. Then we'll have getSuccessOrdering, getFailureOrdering, and getMergedOrdering. That's much clearer. (And, yes, this does still make sense for non-cmpxchg -- they always succeed, so the success ordering is the only relevant thing for them.)

efriedma added inline comments.May 28 2021, 12:15 PM
llvm/include/llvm/CodeGen/MachineMemOperand.h
261

So the idea is basically just rename getOrdering()->getSuccessOrdering()? Sure, I'll look at doing that in a followup.

This revision was landed with ongoing or failed builds.May 28 2021, 12:48 PM
Closed by commit rG0b3b0a727ad6: [AArch64][RISCV] Make sure isel correctly honors failure orderings. (authored by efriedma). · Explain Why
This revision was automatically updated to reflect the committed changes.
efriedma added a commit: rG0b3b0a727ad6: [AArch64][RISCV] Make sure isel correctly honors failure orderings..
efriedma mentioned this in D106729: [AMDGPU] Support non-strictly stronger memory orderings in SIMemoryLegalizer.Jul 26 2021, 10:42 AM

Revision Contents

PathSize
llvm/
include/
llvm/
CodeGen/
GlobalISel/
6 lines
17 lines
5 lines
lib/
CodeGen/
SelectionDAG/
2 lines
2 lines
Target/
AArch64/
2 lines
GISel/
2 lines
test/
CodeGen/
AArch64/
GlobalISel/
46 lines
212 lines
691 lines
RISCV/
6 lines
utils/
TableGen/
18 lines

Diff 348574

llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

Show First 20 Lines • Show All 384 Lines • ▼ Show 20 Lines case GIM_CheckAtomicOrdering: {
dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
<< InsnID << "], " << (uint64_t)Ordering << ")\n"); << InsnID << "], " << (uint64_t)Ordering << ")\n");
assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
if (!State.MIs[InsnID]->hasOneMemOperand()) if (!State.MIs[InsnID]->hasOneMemOperand())
if (handleReject() == RejectAndGiveUp) if (handleReject() == RejectAndGiveUp)
return false; return false;
for (const auto &MMO : State.MIs[InsnID]->memoperands()) for (const auto &MMO : State.MIs[InsnID]->memoperands())
if (MMO->getOrdering() != Ordering) if (MMO->getMergedOrdering() != Ordering)
if (handleReject() == RejectAndGiveUp) if (handleReject() == RejectAndGiveUp)
return false; return false;
break; break;
} }
case GIM_CheckAtomicOrderingOrStrongerThan: { case GIM_CheckAtomicOrderingOrStrongerThan: {
int64_t InsnID = MatchTable[CurrentIdx++]; int64_t InsnID = MatchTable[CurrentIdx++];
AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
dbgs() << CurrentIdx dbgs() << CurrentIdx
<< ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
<< InsnID << "], " << (uint64_t)Ordering << ")\n"); << InsnID << "], " << (uint64_t)Ordering << ")\n");
assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
if (!State.MIs[InsnID]->hasOneMemOperand()) if (!State.MIs[InsnID]->hasOneMemOperand())
if (handleReject() == RejectAndGiveUp) if (handleReject() == RejectAndGiveUp)
return false; return false;
for (const auto &MMO : State.MIs[InsnID]->memoperands()) for (const auto &MMO : State.MIs[InsnID]->memoperands())
if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
if (handleReject() == RejectAndGiveUp) if (handleReject() == RejectAndGiveUp)
return false; return false;
break; break;
} }
case GIM_CheckAtomicOrderingWeakerThan: { case GIM_CheckAtomicOrderingWeakerThan: {
int64_t InsnID = MatchTable[CurrentIdx++]; int64_t InsnID = MatchTable[CurrentIdx++];
AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
dbgs() << CurrentIdx dbgs() << CurrentIdx
<< ": GIM_CheckAtomicOrderingWeakerThan(MIs[" << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
<< InsnID << "], " << (uint64_t)Ordering << ")\n"); << InsnID << "], " << (uint64_t)Ordering << ")\n");
assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
if (!State.MIs[InsnID]->hasOneMemOperand()) if (!State.MIs[InsnID]->hasOneMemOperand())
if (handleReject() == RejectAndGiveUp) if (handleReject() == RejectAndGiveUp)
return false; return false;
for (const auto &MMO : State.MIs[InsnID]->memoperands()) for (const auto &MMO : State.MIs[InsnID]->memoperands())
if (!isStrongerThan(Ordering, MMO->getOrdering())) if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
if (handleReject() == RejectAndGiveUp) if (handleReject() == RejectAndGiveUp)
return false; return false;
break; break;
} }
case GIM_CheckMemoryAddressSpace: { case GIM_CheckMemoryAddressSpace: {
int64_t InsnID = MatchTable[CurrentIdx++]; int64_t InsnID = MatchTable[CurrentIdx++];
int64_t MMOIdx = MatchTable[CurrentIdx++]; int64_t MMOIdx = MatchTable[CurrentIdx++];
// This accepts a list of possible address spaces. // This accepts a list of possible address spaces.
▲ Show 20 Lines • Show All 720 Lines • Show Last 20 Lines

llvm/include/llvm/CodeGen/MachineMemOperand.h

Show First 20 Lines • Show All 249 Lines • ▼ Show 20 Linespublic:
} }
/// For cmpxchg atomic operations, return the atomic ordering requirements /// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur. /// when store does not occur.
AtomicOrdering getFailureOrdering() const { AtomicOrdering getFailureOrdering() const {
return static_cast<AtomicOrdering>(AtomicInfo.FailureOrdering); return static_cast<AtomicOrdering>(AtomicInfo.FailureOrdering);
} }
/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation. (For operations
/// other than cmpxchg, this is equivalent to getOrdering().)
AtomicOrdering getMergedOrdering() const {
jyknightUnsubmitted
Not Done
ReplyInline Actions

I think this is a confusing API now. The reason it made sense to have getOrdering return the success ordering for a cmpxchg before is because that was always stronger than the failure ordering. So, using the success order was a safe "default" ordering. That's no longer the case.

Maybe we should have getOrdering should return the merged ordering, and rename the existing getOrdering to getSuccessOrdering.

Or, actually I like this better: remove getOrdering entirely, by renaming to getSuccessOrdering. Then we'll have getSuccessOrdering, getFailureOrdering, and getMergedOrdering. That's much clearer. (And, yes, this does still make sense for non-cmpxchg -- they always succeed, so the success ordering is the only relevant thing for them.)

jyknight: I think this is a confusing API now. The reason it made sense to have getOrdering return the…
efriedmaAuthorUnsubmitted
Done
ReplyInline Actions

So the idea is basically just rename getOrdering()->getSuccessOrdering()? Sure, I'll look at doing that in a followup.

efriedma: So the idea is basically just rename getOrdering()->getSuccessOrdering()? Sure, I'll look at…
AtomicOrdering Ordering = getOrdering();
AtomicOrdering FailureOrdering = getFailureOrdering();
if (FailureOrdering == AtomicOrdering::SequentiallyConsistent)
return AtomicOrdering::SequentiallyConsistent;
if (FailureOrdering == AtomicOrdering::Acquire) {
if (Ordering == AtomicOrdering::Monotonic)
return AtomicOrdering::Acquire;
if (Ordering == AtomicOrdering::Release)
return AtomicOrdering::AcquireRelease;
}
return Ordering;
}
bool isLoad() const { return FlagVals & MOLoad; } bool isLoad() const { return FlagVals & MOLoad; }
bool isStore() const { return FlagVals & MOStore; } bool isStore() const { return FlagVals & MOStore; }
bool isVolatile() const { return FlagVals & MOVolatile; } bool isVolatile() const { return FlagVals & MOVolatile; }
bool isNonTemporal() const { return FlagVals & MONonTemporal; } bool isNonTemporal() const { return FlagVals & MONonTemporal; }
bool isDereferenceable() const { return FlagVals & MODereferenceable; } bool isDereferenceable() const { return FlagVals & MODereferenceable; }
bool isInvariant() const { return FlagVals & MOInvariant; } bool isInvariant() const { return FlagVals & MOInvariant; }
/// Returns true if this operation has an atomic ordering requirement of /// Returns true if this operation has an atomic ordering requirement of
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Show First 20 Lines • Show All 1,300 Lines • ▼ Show 20 Linespublic:
/// Returns the synchronization scope ID for this memory operation. /// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
/// Return the atomic ordering requirements for this memory operation. For /// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when /// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs. /// store occurs.
AtomicOrdering getOrdering() const { return MMO->getOrdering(); } AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation. (For operations
/// other than cmpxchg, this is equivalent to getOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
/// Return true if the memory operation ordering is Unordered or higher. /// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); } bool isAtomic() const { return MMO->isAtomic(); }
/// Returns true if the memory operation doesn't imply any ordering /// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory /// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules. /// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); } bool isUnordered() const { return MMO->isUnordered(); }
▲ Show 20 Lines • Show All 1,439 Lines • Show Last 20 Lines

llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Show First 20 Lines • Show All 3,889 Lines • ▼ Show 20 Linesvoid SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
case ISD::ATOMIC_LOAD_XOR: case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND: case ISD::ATOMIC_LOAD_NAND:
case ISD::ATOMIC_LOAD_MIN: case ISD::ATOMIC_LOAD_MIN:
case ISD::ATOMIC_LOAD_MAX: case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN: case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX: case ISD::ATOMIC_LOAD_UMAX:
case ISD::ATOMIC_CMP_SWAP: { case ISD::ATOMIC_CMP_SWAP: {
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
AtomicOrdering Order = cast<AtomicSDNode>(Node)->getOrdering(); AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT); RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
EVT RetVT = Node->getValueType(0); EVT RetVT = Node->getValueType(0);
TargetLowering::MakeLibCallOptions CallOptions; TargetLowering::MakeLibCallOptions CallOptions;
SmallVector<SDValue, 4> Ops; SmallVector<SDValue, 4> Ops;
if (TLI.getLibcallName(LC)) { if (TLI.getLibcallName(LC)) {
// If outline atomic available, prepare its arguments and expand. // If outline atomic available, prepare its arguments and expand.
Ops.append(Node->op_begin() + 2, Node->op_end()); Ops.append(Node->op_begin() + 2, Node->op_end());
Ops.push_back(Node->getOperand(1)); Ops.push_back(Node->getOperand(1));
▲ Show 20 Lines • Show All 1,098 Lines • Show Last 20 Lines

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Show First 20 Lines • Show All 2,219 Lines • ▼ Show 20 Lines#endif
if (Lo.getNode()) if (Lo.getNode())
SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
} }
/// Lower an atomic node to the appropriate builtin call. /// Lower an atomic node to the appropriate builtin call.
std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
unsigned Opc = Node->getOpcode(); unsigned Opc = Node->getOpcode();
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
AtomicOrdering order = cast<AtomicSDNode>(Node)->getOrdering(); AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
// Lower to outline atomic libcall if outline atomics enabled, // Lower to outline atomic libcall if outline atomics enabled,
// or to sync libcall otherwise // or to sync libcall otherwise
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
EVT RetVT = Node->getValueType(0); EVT RetVT = Node->getValueType(0);
TargetLowering::MakeLibCallOptions CallOptions; TargetLowering::MakeLibCallOptions CallOptions;
SmallVector<SDValue, 4> Ops; SmallVector<SDValue, 4> Ops;
if (TLI.getLibcallName(LC)) { if (TLI.getLibcallName(LC)) {
Ops.append(Node->op_begin() + 2, Node->op_end()); Ops.append(Node->op_begin() + 2, Node->op_end());
▲ Show 20 Lines • Show All 2,720 Lines • Show Last 20 Lines

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 16,669 Lines • ▼ Show 20 Lines SDValue Ops[] = {
createGPRPairNode(DAG, N->getOperand(3)), // Store value createGPRPairNode(DAG, N->getOperand(3)), // Store value
N->getOperand(1), // Ptr N->getOperand(1), // Ptr
N->getOperand(0), // Chain in N->getOperand(0), // Chain in
}; };
MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
unsigned Opcode; unsigned Opcode;
switch (MemOp->getOrdering()) { switch (MemOp->getMergedOrdering()) {
case AtomicOrdering::Monotonic: case AtomicOrdering::Monotonic:
Opcode = AArch64::CASPX; Opcode = AArch64::CASPX;
break; break;
case AtomicOrdering::Acquire: case AtomicOrdering::Acquire:
Opcode = AArch64::CASPAX; Opcode = AArch64::CASPAX;
break; break;
case AtomicOrdering::Release: case AtomicOrdering::Release:
Opcode = AArch64::CASPLX; Opcode = AArch64::CASPLX;
▲ Show 20 Lines • Show All 1,495 Lines • Show Last 20 Lines

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

Show First 20 Lines • Show All 1,090 Lines • ▼ Show 20 Lines if (ST->hasLSE()) {
// s128. We need the merge/unmerge to bracket the expansion and pair up with // s128. We need the merge/unmerge to bracket the expansion and pair up with
// the rest of the MIR so we must reassemble the extracted registers into a // the rest of the MIR so we must reassemble the extracted registers into a
// 128-bit known-regclass one with code like this: // 128-bit known-regclass one with code like this:
// //
// %in1 = REG_SEQUENCE Lo, Hi ; One for each input // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
// %out = CASP %in1, ... // %out = CASP %in1, ...
// %OldLo = G_EXTRACT %out, 0 // %OldLo = G_EXTRACT %out, 0
// %OldHi = G_EXTRACT %out, 64 // %OldHi = G_EXTRACT %out, 64
auto Ordering = (*MI.memoperands_begin())->getOrdering(); auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
unsigned Opcode; unsigned Opcode;
switch (Ordering) { switch (Ordering) {
case AtomicOrdering::Acquire: case AtomicOrdering::Acquire:
Opcode = AArch64::CASPAX; Opcode = AArch64::CASPAX;
break; break;
case AtomicOrdering::Release: case AtomicOrdering::Release:
Opcode = AArch64::CASPLX; Opcode = AArch64::CASPLX;
break; break;
▲ Show 20 Lines • Show All 48 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll

Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines
; CHECK-CAS-O0: mov v[[OLD]].d[1], [[OLD_HI]] ; CHECK-CAS-O0: mov v[[OLD]].d[1], [[OLD_HI]]
; CHECK-CAS-O0: str q[[OLD]], [x0] ; CHECK-CAS-O0: str q[[OLD]], [x0]
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
%val = extractvalue { i128, i1 } %pair, 0 %val = extractvalue { i128, i1 } %pair, 0
store i128 %val, i128* %p store i128 %val, i128* %p
ret void ret void
} }
define void @val_compare_and_swap_monotonic_seqcst(i128* %p, i128 %oldval, i128 %newval) {
; CHECK-LLSC-O1-LABEL: val_compare_and_swap_monotonic_seqcst:
; CHECK-LLSC-O1: ldxp {{x[0-9]+}}, {{x[0-9]+}}, [x0]
; [... LOTS of stuff that is generic IR unrelated to atomic operations ...]
; CHECK-LLSC-O1: stxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0]
;
; CHECK-CAS-O1-LABEL: val_compare_and_swap_monotonic_seqcst:
; CHECK-CAS-O1: caspal x2, x3, x4, x5, [x0]
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic_seqcst:
; CHECK-LLSC-O0: .LBB1_1:
; CHECK-LLSC-O0: ldaxp
; CHECK-LLSC-O0: stlxp
; CHECK-CAS-O0-LABEL: val_compare_and_swap_monotonic_seqcst:
; CHECK-CAS-O0: caspal
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval monotonic seq_cst
%val = extractvalue { i128, i1 } %pair, 0
store i128 %val, i128* %p
ret void
}
define void @val_compare_and_swap_release_acquire(i128* %p, i128 %oldval, i128 %newval) {
; CHECK-LLSC-O1-LABEL: val_compare_and_swap_release_acquire:
; CHECK-LLSC-O1: ldxp {{x[0-9]+}}, {{x[0-9]+}}, [x0]
; [... LOTS of stuff that is generic IR unrelated to atomic operations ...]
; CHECK-LLSC-O1: stlxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0]
;
; CHECK-CAS-O1-LABEL: val_compare_and_swap_release_acquire:
; CHECK-CAS-O1: caspal x2, x3, x4, x5, [x0]
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_release_acquire:
; CHECK-LLSC-O0: .LBB2_1:
; CHECK-LLSC-O0: ldaxp
; CHECK-LLSC-O0: stlxp
; CHECK-CAS-O0-LABEL: val_compare_and_swap_release_acquire:
; CHECK-CAS-O0: caspal
%pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire
%val = extractvalue { i128, i1 } %pair, 0
store i128 %val, i128* %p
ret void
}

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic.ll

Show First 20 Lines • Show All 203 Lines • ▼ Show 20 Lines
; CHECK-LSE-O0-NEXT: mov x0, x1 ; CHECK-LSE-O0-NEXT: mov x0, x1
; CHECK-LSE-O0-NEXT: cas x0, x2, [x8] ; CHECK-LSE-O0-NEXT: cas x0, x2, [x8]
; CHECK-LSE-O0-NEXT: ret ; CHECK-LSE-O0-NEXT: ret
%pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic monotonic %pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic monotonic
%val = extractvalue { i64, i1 } %pair, 0 %val = extractvalue { i64, i1 } %pair, 0
ret i64 %val ret i64 %val
} }
define i64 @val_compare_and_swap_64_monotonic_seqcst(i64* %p, i64 %cmp, i64 %new) #0 {
; CHECK-NOLSE-O1-LABEL: val_compare_and_swap_64_monotonic_seqcst:
; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: LBB4_1: ; %cmpxchg.start
; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O1-NEXT: ldxr x8, [x0]
; CHECK-NOLSE-O1-NEXT: cmp x8, x1
; CHECK-NOLSE-O1-NEXT: b.ne LBB4_4
; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %cmpxchg.trystore
; CHECK-NOLSE-O1-NEXT: ; in Loop: Header=BB4_1 Depth=1
; CHECK-NOLSE-O1-NEXT: stxr w9, x2, [x0]
; CHECK-NOLSE-O1-NEXT: cbnz w9, LBB4_1
; CHECK-NOLSE-O1-NEXT: ; %bb.3: ; %cmpxchg.end
; CHECK-NOLSE-O1-NEXT: mov x0, x8
; CHECK-NOLSE-O1-NEXT: ret
; CHECK-NOLSE-O1-NEXT: LBB4_4: ; %cmpxchg.nostore
; CHECK-NOLSE-O1-NEXT: clrex
; CHECK-NOLSE-O1-NEXT: mov x0, x8
; CHECK-NOLSE-O1-NEXT: ret
;
; CHECK-NOLSE-O0-LABEL: val_compare_and_swap_64_monotonic_seqcst:
; CHECK-NOLSE-O0: ; %bb.0:
; CHECK-NOLSE-O0-NEXT: mov x9, x0
; CHECK-NOLSE-O0-NEXT: LBB4_1: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O0-NEXT: ldaxr x0, [x9]
; CHECK-NOLSE-O0-NEXT: cmp x0, x1
; CHECK-NOLSE-O0-NEXT: b.ne LBB4_3
; CHECK-NOLSE-O0-NEXT: ; %bb.2: ; in Loop: Header=BB4_1 Depth=1
; CHECK-NOLSE-O0-NEXT: stlxr w8, x2, [x9]
; CHECK-NOLSE-O0-NEXT: cbnz w8, LBB4_1
; CHECK-NOLSE-O0-NEXT: LBB4_3:
; CHECK-NOLSE-O0-NEXT: ret
;
; CHECK-LSE-O1-LABEL: val_compare_and_swap_64_monotonic_seqcst:
; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: casal x1, x2, [x0]
; CHECK-LSE-O1-NEXT: mov x0, x1
; CHECK-LSE-O1-NEXT: ret
;
; CHECK-LSE-O0-LABEL: val_compare_and_swap_64_monotonic_seqcst:
; CHECK-LSE-O0: ; %bb.0:
; CHECK-LSE-O0-NEXT: mov x8, x0
; CHECK-LSE-O0-NEXT: mov x0, x1
; CHECK-LSE-O0-NEXT: casal x0, x2, [x8]
; CHECK-LSE-O0-NEXT: ret
%pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic seq_cst
%val = extractvalue { i64, i1 } %pair, 0
ret i64 %val
}
define i64 @val_compare_and_swap_64_release_acquire(i64* %p, i64 %cmp, i64 %new) #0 {
; CHECK-NOLSE-O1-LABEL: val_compare_and_swap_64_release_acquire:
; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: LBB5_1: ; %cmpxchg.start
; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O1-NEXT: ldxr x8, [x0]
; CHECK-NOLSE-O1-NEXT: cmp x8, x1
; CHECK-NOLSE-O1-NEXT: b.ne LBB5_4
; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %cmpxchg.trystore
; CHECK-NOLSE-O1-NEXT: ; in Loop: Header=BB5_1 Depth=1
; CHECK-NOLSE-O1-NEXT: stlxr w9, x2, [x0]
; CHECK-NOLSE-O1-NEXT: cbnz w9, LBB5_1
; CHECK-NOLSE-O1-NEXT: ; %bb.3: ; %cmpxchg.end
; CHECK-NOLSE-O1-NEXT: mov x0, x8
; CHECK-NOLSE-O1-NEXT: ret
; CHECK-NOLSE-O1-NEXT: LBB5_4: ; %cmpxchg.nostore
; CHECK-NOLSE-O1-NEXT: clrex
; CHECK-NOLSE-O1-NEXT: mov x0, x8
; CHECK-NOLSE-O1-NEXT: ret
;
; CHECK-NOLSE-O0-LABEL: val_compare_and_swap_64_release_acquire:
; CHECK-NOLSE-O0: ; %bb.0:
; CHECK-NOLSE-O0-NEXT: mov x9, x0
; CHECK-NOLSE-O0-NEXT: LBB5_1: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O0-NEXT: ldaxr x0, [x9]
; CHECK-NOLSE-O0-NEXT: cmp x0, x1
; CHECK-NOLSE-O0-NEXT: b.ne LBB5_3
; CHECK-NOLSE-O0-NEXT: ; %bb.2: ; in Loop: Header=BB5_1 Depth=1
; CHECK-NOLSE-O0-NEXT: stlxr w8, x2, [x9]
; CHECK-NOLSE-O0-NEXT: cbnz w8, LBB5_1
; CHECK-NOLSE-O0-NEXT: LBB5_3:
; CHECK-NOLSE-O0-NEXT: ret
;
; CHECK-LSE-O1-LABEL: val_compare_and_swap_64_release_acquire:
; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: casal x1, x2, [x0]
; CHECK-LSE-O1-NEXT: mov x0, x1
; CHECK-LSE-O1-NEXT: ret
;
; CHECK-LSE-O0-LABEL: val_compare_and_swap_64_release_acquire:
; CHECK-LSE-O0: ; %bb.0:
; CHECK-LSE-O0-NEXT: mov x8, x0
; CHECK-LSE-O0-NEXT: mov x0, x1
; CHECK-LSE-O0-NEXT: casal x0, x2, [x8]
; CHECK-LSE-O0-NEXT: ret
%pair = cmpxchg i64* %p, i64 %cmp, i64 %new release acquire
%val = extractvalue { i64, i1 } %pair, 0
ret i64 %val
}
define i32 @fetch_and_nand(i32* %p) #0 { define i32 @fetch_and_nand(i32* %p) #0 {
; CHECK-NOLSE-O1-LABEL: fetch_and_nand: ; CHECK-NOLSE-O1-LABEL: fetch_and_nand:
; CHECK-NOLSE-O1: ; %bb.0: ; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: LBB4_1: ; %atomicrmw.start ; CHECK-NOLSE-O1-NEXT: LBB6_1: ; %atomicrmw.start
; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O1-NEXT: ldxr w8, [x0] ; CHECK-NOLSE-O1-NEXT: ldxr w8, [x0]
; CHECK-NOLSE-O1-NEXT: and w9, w8, #0x7 ; CHECK-NOLSE-O1-NEXT: and w9, w8, #0x7
; CHECK-NOLSE-O1-NEXT: mvn w9, w9 ; CHECK-NOLSE-O1-NEXT: mvn w9, w9
; CHECK-NOLSE-O1-NEXT: stlxr w10, w9, [x0] ; CHECK-NOLSE-O1-NEXT: stlxr w10, w9, [x0]
; CHECK-NOLSE-O1-NEXT: cbnz w10, LBB4_1 ; CHECK-NOLSE-O1-NEXT: cbnz w10, LBB6_1
; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NOLSE-O1-NEXT: mov w0, w8 ; CHECK-NOLSE-O1-NEXT: mov w0, w8
; CHECK-NOLSE-O1-NEXT: ret ; CHECK-NOLSE-O1-NEXT: ret
; ;
; CHECK-NOLSE-O0-LABEL: fetch_and_nand: ; CHECK-NOLSE-O0-LABEL: fetch_and_nand:
; CHECK-NOLSE-O0: ; %bb.0: ; CHECK-NOLSE-O0: ; %bb.0:
; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: ldr w8, [x0] ; CHECK-NOLSE-O0-NEXT: ldr w8, [x0]
; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: LBB4_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB6_1: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NOLSE-O0-NEXT: ; Child Loop BB4_2 Depth 2 ; CHECK-NOLSE-O0-NEXT: ; Child Loop BB6_2 Depth 2
; CHECK-NOLSE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: and w9, w8, #0x7 ; CHECK-NOLSE-O0-NEXT: and w9, w8, #0x7
; CHECK-NOLSE-O0-NEXT: mvn w12, w9 ; CHECK-NOLSE-O0-NEXT: mvn w12, w9
; CHECK-NOLSE-O0-NEXT: LBB4_2: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB6_2: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB4_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB6_1 Depth=1
; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2 ; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2
; CHECK-NOLSE-O0-NEXT: ldaxr w9, [x11] ; CHECK-NOLSE-O0-NEXT: ldaxr w9, [x11]
; CHECK-NOLSE-O0-NEXT: cmp w9, w8 ; CHECK-NOLSE-O0-NEXT: cmp w9, w8
; CHECK-NOLSE-O0-NEXT: b.ne LBB4_4 ; CHECK-NOLSE-O0-NEXT: b.ne LBB6_4
; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB4_2 Depth=2 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB6_2 Depth=2
; CHECK-NOLSE-O0-NEXT: stlxr w10, w12, [x11] ; CHECK-NOLSE-O0-NEXT: stlxr w10, w12, [x11]
; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB4_2 ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB6_2
; CHECK-NOLSE-O0-NEXT: LBB4_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB6_4: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB4_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB6_1 Depth=1
; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8
; CHECK-NOLSE-O0-NEXT: cset w8, eq ; CHECK-NOLSE-O0-NEXT: cset w8, eq
; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB4_1 ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB6_1
; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end ; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end
; CHECK-NOLSE-O0-NEXT: ldr w0, [sp, #12] ; 4-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr w0, [sp, #12] ; 4-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: ret ; CHECK-NOLSE-O0-NEXT: ret
; ;
; CHECK-LSE-O1-LABEL: fetch_and_nand: ; CHECK-LSE-O1-LABEL: fetch_and_nand:
; CHECK-LSE-O1: ; %bb.0: ; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: LBB4_1: ; %atomicrmw.start ; CHECK-LSE-O1-NEXT: LBB6_1: ; %atomicrmw.start
; CHECK-LSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-LSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-LSE-O1-NEXT: ldxr w8, [x0] ; CHECK-LSE-O1-NEXT: ldxr w8, [x0]
; CHECK-LSE-O1-NEXT: and w9, w8, #0x7 ; CHECK-LSE-O1-NEXT: and w9, w8, #0x7
; CHECK-LSE-O1-NEXT: mvn w9, w9 ; CHECK-LSE-O1-NEXT: mvn w9, w9
; CHECK-LSE-O1-NEXT: stlxr w10, w9, [x0] ; CHECK-LSE-O1-NEXT: stlxr w10, w9, [x0]
; CHECK-LSE-O1-NEXT: cbnz w10, LBB4_1 ; CHECK-LSE-O1-NEXT: cbnz w10, LBB6_1
; CHECK-LSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-LSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-LSE-O1-NEXT: mov x0, x8 ; CHECK-LSE-O1-NEXT: mov x0, x8
; CHECK-LSE-O1-NEXT: ret ; CHECK-LSE-O1-NEXT: ret
; ;
; CHECK-LSE-O0-LABEL: fetch_and_nand: ; CHECK-LSE-O0-LABEL: fetch_and_nand:
; CHECK-LSE-O0: ; %bb.0: ; CHECK-LSE-O0: ; %bb.0:
; CHECK-LSE-O0-NEXT: sub sp, sp, #32 ; =32 ; CHECK-LSE-O0-NEXT: sub sp, sp, #32 ; =32
; CHECK-LSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill
; CHECK-LSE-O0-NEXT: ldr w8, [x0] ; CHECK-LSE-O0-NEXT: ldr w8, [x0]
; CHECK-LSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Folded Spill ; CHECK-LSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Folded Spill
; CHECK-LSE-O0-NEXT: LBB4_1: ; %atomicrmw.start ; CHECK-LSE-O0-NEXT: LBB6_1: ; %atomicrmw.start
; CHECK-LSE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-LSE-O0-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-LSE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Folded Reload ; CHECK-LSE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Folded Reload
; CHECK-LSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload ; CHECK-LSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload
; CHECK-LSE-O0-NEXT: and w9, w8, #0x7 ; CHECK-LSE-O0-NEXT: and w9, w8, #0x7
; CHECK-LSE-O0-NEXT: mvn w10, w9 ; CHECK-LSE-O0-NEXT: mvn w10, w9
; CHECK-LSE-O0-NEXT: mov x9, x8 ; CHECK-LSE-O0-NEXT: mov x9, x8
; CHECK-LSE-O0-NEXT: casl w9, w10, [x11] ; CHECK-LSE-O0-NEXT: casl w9, w10, [x11]
; CHECK-LSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-LSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill
; CHECK-LSE-O0-NEXT: subs w8, w9, w8 ; CHECK-LSE-O0-NEXT: subs w8, w9, w8
; CHECK-LSE-O0-NEXT: cset w8, eq ; CHECK-LSE-O0-NEXT: cset w8, eq
; CHECK-LSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-LSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill
; CHECK-LSE-O0-NEXT: tbz w8, #0, LBB4_1 ; CHECK-LSE-O0-NEXT: tbz w8, #0, LBB6_1
; CHECK-LSE-O0-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-LSE-O0-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-LSE-O0-NEXT: ldr w0, [sp, #12] ; 4-byte Folded Reload ; CHECK-LSE-O0-NEXT: ldr w0, [sp, #12] ; 4-byte Folded Reload
; CHECK-LSE-O0-NEXT: add sp, sp, #32 ; =32 ; CHECK-LSE-O0-NEXT: add sp, sp, #32 ; =32
; CHECK-LSE-O0-NEXT: ret ; CHECK-LSE-O0-NEXT: ret
%val = atomicrmw nand i32* %p, i32 7 release %val = atomicrmw nand i32* %p, i32 7 release
ret i32 %val ret i32 %val
} }
define i64 @fetch_and_nand_64(i64* %p) #0 { define i64 @fetch_and_nand_64(i64* %p) #0 {
; CHECK-NOLSE-O1-LABEL: fetch_and_nand_64: ; CHECK-NOLSE-O1-LABEL: fetch_and_nand_64:
; CHECK-NOLSE-O1: ; %bb.0: ; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: LBB5_1: ; %atomicrmw.start ; CHECK-NOLSE-O1-NEXT: LBB7_1: ; %atomicrmw.start
; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O1-NEXT: ldaxr x8, [x0] ; CHECK-NOLSE-O1-NEXT: ldaxr x8, [x0]
; CHECK-NOLSE-O1-NEXT: and x9, x8, #0x7 ; CHECK-NOLSE-O1-NEXT: and x9, x8, #0x7
; CHECK-NOLSE-O1-NEXT: mvn x9, x9 ; CHECK-NOLSE-O1-NEXT: mvn x9, x9
; CHECK-NOLSE-O1-NEXT: stlxr w10, x9, [x0] ; CHECK-NOLSE-O1-NEXT: stlxr w10, x9, [x0]
; CHECK-NOLSE-O1-NEXT: cbnz w10, LBB5_1 ; CHECK-NOLSE-O1-NEXT: cbnz w10, LBB7_1
; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NOLSE-O1-NEXT: mov x0, x8 ; CHECK-NOLSE-O1-NEXT: mov x0, x8
; CHECK-NOLSE-O1-NEXT: ret ; CHECK-NOLSE-O1-NEXT: ret
; ;
; CHECK-NOLSE-O0-LABEL: fetch_and_nand_64: ; CHECK-NOLSE-O0-LABEL: fetch_and_nand_64:
; CHECK-NOLSE-O0: ; %bb.0: ; CHECK-NOLSE-O0: ; %bb.0:
; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: ldr x8, [x0] ; CHECK-NOLSE-O0-NEXT: ldr x8, [x0]
; CHECK-NOLSE-O0-NEXT: str x8, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x8, [sp, #24] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: LBB5_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB7_1: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NOLSE-O0-NEXT: ; Child Loop BB5_2 Depth 2 ; CHECK-NOLSE-O0-NEXT: ; Child Loop BB7_2 Depth 2
; CHECK-NOLSE-O0-NEXT: ldr x8, [sp, #24] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x8, [sp, #24] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: and x9, x8, #0x7 ; CHECK-NOLSE-O0-NEXT: and x9, x8, #0x7
; CHECK-NOLSE-O0-NEXT: mvn x12, x9 ; CHECK-NOLSE-O0-NEXT: mvn x12, x9
; CHECK-NOLSE-O0-NEXT: LBB5_2: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB7_2: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB5_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB7_1 Depth=1
; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2 ; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2
; CHECK-NOLSE-O0-NEXT: ldaxr x9, [x11] ; CHECK-NOLSE-O0-NEXT: ldaxr x9, [x11]
; CHECK-NOLSE-O0-NEXT: cmp x9, x8 ; CHECK-NOLSE-O0-NEXT: cmp x9, x8
; CHECK-NOLSE-O0-NEXT: b.ne LBB5_4 ; CHECK-NOLSE-O0-NEXT: b.ne LBB7_4
; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB5_2 Depth=2 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB7_2 Depth=2
; CHECK-NOLSE-O0-NEXT: stlxr w10, x12, [x11] ; CHECK-NOLSE-O0-NEXT: stlxr w10, x12, [x11]
; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB5_2 ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB7_2
; CHECK-NOLSE-O0-NEXT: LBB5_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB7_4: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB7_1 Depth=1
; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8
; CHECK-NOLSE-O0-NEXT: cset w8, eq ; CHECK-NOLSE-O0-NEXT: cset w8, eq
; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB5_1 ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB7_1
; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end ; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end
; CHECK-NOLSE-O0-NEXT: ldr x0, [sp, #8] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x0, [sp, #8] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: ret ; CHECK-NOLSE-O0-NEXT: ret
; ;
; CHECK-LSE-O1-LABEL: fetch_and_nand_64: ; CHECK-LSE-O1-LABEL: fetch_and_nand_64:
; CHECK-LSE-O1: ; %bb.0: ; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: LBB5_1: ; %atomicrmw.start ; CHECK-LSE-O1-NEXT: LBB7_1: ; %atomicrmw.start
; CHECK-LSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-LSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-LSE-O1-NEXT: ldaxr x8, [x0] ; CHECK-LSE-O1-NEXT: ldaxr x8, [x0]
; CHECK-LSE-O1-NEXT: and x9, x8, #0x7 ; CHECK-LSE-O1-NEXT: and x9, x8, #0x7
; CHECK-LSE-O1-NEXT: mvn x9, x9 ; CHECK-LSE-O1-NEXT: mvn x9, x9
; CHECK-LSE-O1-NEXT: stlxr w10, x9, [x0] ; CHECK-LSE-O1-NEXT: stlxr w10, x9, [x0]
; CHECK-LSE-O1-NEXT: cbnz w10, LBB5_1 ; CHECK-LSE-O1-NEXT: cbnz w10, LBB7_1
; CHECK-LSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-LSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-LSE-O1-NEXT: mov x0, x8 ; CHECK-LSE-O1-NEXT: mov x0, x8
; CHECK-LSE-O1-NEXT: ret ; CHECK-LSE-O1-NEXT: ret
; ;
; CHECK-LSE-O0-LABEL: fetch_and_nand_64: ; CHECK-LSE-O0-LABEL: fetch_and_nand_64:
; CHECK-LSE-O0: ; %bb.0: ; CHECK-LSE-O0: ; %bb.0:
; CHECK-LSE-O0-NEXT: sub sp, sp, #32 ; =32 ; CHECK-LSE-O0-NEXT: sub sp, sp, #32 ; =32
; CHECK-LSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill
; CHECK-LSE-O0-NEXT: ldr x8, [x0] ; CHECK-LSE-O0-NEXT: ldr x8, [x0]
; CHECK-LSE-O0-NEXT: str x8, [sp, #24] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: str x8, [sp, #24] ; 8-byte Folded Spill
; CHECK-LSE-O0-NEXT: LBB5_1: ; %atomicrmw.start ; CHECK-LSE-O0-NEXT: LBB7_1: ; %atomicrmw.start
; CHECK-LSE-O0-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-LSE-O0-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-LSE-O0-NEXT: ldr x8, [sp, #24] ; 8-byte Folded Reload ; CHECK-LSE-O0-NEXT: ldr x8, [sp, #24] ; 8-byte Folded Reload
; CHECK-LSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload ; CHECK-LSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload
; CHECK-LSE-O0-NEXT: and x9, x8, #0x7 ; CHECK-LSE-O0-NEXT: and x9, x8, #0x7
; CHECK-LSE-O0-NEXT: mvn x10, x9 ; CHECK-LSE-O0-NEXT: mvn x10, x9
; CHECK-LSE-O0-NEXT: mov x9, x8 ; CHECK-LSE-O0-NEXT: mov x9, x8
; CHECK-LSE-O0-NEXT: casal x9, x10, [x11] ; CHECK-LSE-O0-NEXT: casal x9, x10, [x11]
; CHECK-LSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill
; CHECK-LSE-O0-NEXT: subs x8, x9, x8 ; CHECK-LSE-O0-NEXT: subs x8, x9, x8
; CHECK-LSE-O0-NEXT: cset w8, eq ; CHECK-LSE-O0-NEXT: cset w8, eq
; CHECK-LSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-LSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill
; CHECK-LSE-O0-NEXT: tbz w8, #0, LBB5_1 ; CHECK-LSE-O0-NEXT: tbz w8, #0, LBB7_1
; CHECK-LSE-O0-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-LSE-O0-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-LSE-O0-NEXT: ldr x0, [sp, #8] ; 8-byte Folded Reload ; CHECK-LSE-O0-NEXT: ldr x0, [sp, #8] ; 8-byte Folded Reload
; CHECK-LSE-O0-NEXT: add sp, sp, #32 ; =32 ; CHECK-LSE-O0-NEXT: add sp, sp, #32 ; =32
; CHECK-LSE-O0-NEXT: ret ; CHECK-LSE-O0-NEXT: ret
%val = atomicrmw nand i64* %p, i64 7 acq_rel %val = atomicrmw nand i64* %p, i64 7 acq_rel
ret i64 %val ret i64 %val
} }
define i32 @fetch_and_or(i32* %p) #0 { define i32 @fetch_and_or(i32* %p) #0 {
; CHECK-NOLSE-O1-LABEL: fetch_and_or: ; CHECK-NOLSE-O1-LABEL: fetch_and_or:
; CHECK-NOLSE-O1: ; %bb.0: ; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: mov w9, #5 ; CHECK-NOLSE-O1-NEXT: mov w9, #5
; CHECK-NOLSE-O1-NEXT: LBB6_1: ; %atomicrmw.start ; CHECK-NOLSE-O1-NEXT: LBB8_1: ; %atomicrmw.start
; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O1-NEXT: ldaxr w8, [x0] ; CHECK-NOLSE-O1-NEXT: ldaxr w8, [x0]
; CHECK-NOLSE-O1-NEXT: orr w10, w8, w9 ; CHECK-NOLSE-O1-NEXT: orr w10, w8, w9
; CHECK-NOLSE-O1-NEXT: stlxr w11, w10, [x0] ; CHECK-NOLSE-O1-NEXT: stlxr w11, w10, [x0]
; CHECK-NOLSE-O1-NEXT: cbnz w11, LBB6_1 ; CHECK-NOLSE-O1-NEXT: cbnz w11, LBB8_1
; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NOLSE-O1-NEXT: mov w0, w8 ; CHECK-NOLSE-O1-NEXT: mov w0, w8
; CHECK-NOLSE-O1-NEXT: ret ; CHECK-NOLSE-O1-NEXT: ret
; ;
; CHECK-NOLSE-O0-LABEL: fetch_and_or: ; CHECK-NOLSE-O0-LABEL: fetch_and_or:
; CHECK-NOLSE-O0: ; %bb.0: ; CHECK-NOLSE-O0: ; %bb.0:
; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: ldr w8, [x0] ; CHECK-NOLSE-O0-NEXT: ldr w8, [x0]
; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w8, [sp, #28] ; 4-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: LBB6_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB8_1: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NOLSE-O0-NEXT: ; Child Loop BB6_2 Depth 2 ; CHECK-NOLSE-O0-NEXT: ; Child Loop BB8_2 Depth 2
; CHECK-NOLSE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr w8, [sp, #28] ; 4-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: mov w9, #5 ; CHECK-NOLSE-O0-NEXT: mov w9, #5
; CHECK-NOLSE-O0-NEXT: orr w12, w8, w9 ; CHECK-NOLSE-O0-NEXT: orr w12, w8, w9
; CHECK-NOLSE-O0-NEXT: LBB6_2: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB8_2: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB6_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB8_1 Depth=1
; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2 ; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2
; CHECK-NOLSE-O0-NEXT: ldaxr w9, [x11] ; CHECK-NOLSE-O0-NEXT: ldaxr w9, [x11]
; CHECK-NOLSE-O0-NEXT: cmp w9, w8 ; CHECK-NOLSE-O0-NEXT: cmp w9, w8
; CHECK-NOLSE-O0-NEXT: b.ne LBB6_4 ; CHECK-NOLSE-O0-NEXT: b.ne LBB8_4
; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB6_2 Depth=2 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB8_2 Depth=2
; CHECK-NOLSE-O0-NEXT: stlxr w10, w12, [x11] ; CHECK-NOLSE-O0-NEXT: stlxr w10, w12, [x11]
; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB6_2 ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB8_2
; CHECK-NOLSE-O0-NEXT: LBB6_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB8_4: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB6_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB8_1 Depth=1
; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #12] ; 4-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8 ; CHECK-NOLSE-O0-NEXT: subs w8, w9, w8
; CHECK-NOLSE-O0-NEXT: cset w8, eq ; CHECK-NOLSE-O0-NEXT: cset w8, eq
; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str w9, [sp, #28] ; 4-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB6_1 ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB8_1
; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end ; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end
; CHECK-NOLSE-O0-NEXT: ldr w0, [sp, #12] ; 4-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr w0, [sp, #12] ; 4-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: ret ; CHECK-NOLSE-O0-NEXT: ret
; ;
; CHECK-LSE-O1-LABEL: fetch_and_or: ; CHECK-LSE-O1-LABEL: fetch_and_or:
; CHECK-LSE-O1: ; %bb.0: ; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: mov w8, #5 ; CHECK-LSE-O1-NEXT: mov w8, #5
; CHECK-LSE-O1-NEXT: ldsetal w8, w0, [x0] ; CHECK-LSE-O1-NEXT: ldsetal w8, w0, [x0]
; CHECK-LSE-O1-NEXT: ret ; CHECK-LSE-O1-NEXT: ret
; ;
; CHECK-LSE-O0-LABEL: fetch_and_or: ; CHECK-LSE-O0-LABEL: fetch_and_or:
; CHECK-LSE-O0: ; %bb.0: ; CHECK-LSE-O0: ; %bb.0:
; CHECK-LSE-O0-NEXT: mov w8, #5 ; CHECK-LSE-O0-NEXT: mov w8, #5
; CHECK-LSE-O0-NEXT: ldsetal w8, w0, [x0] ; CHECK-LSE-O0-NEXT: ldsetal w8, w0, [x0]
; CHECK-LSE-O0-NEXT: ret ; CHECK-LSE-O0-NEXT: ret
%val = atomicrmw or i32* %p, i32 5 seq_cst %val = atomicrmw or i32* %p, i32 5 seq_cst
ret i32 %val ret i32 %val
} }
define i64 @fetch_and_or_64(i64* %p) #0 { define i64 @fetch_and_or_64(i64* %p) #0 {
; CHECK-NOLSE-O1-LABEL: fetch_and_or_64: ; CHECK-NOLSE-O1-LABEL: fetch_and_or_64:
; CHECK-NOLSE-O1: ; %bb.0: ; CHECK-NOLSE-O1: ; %bb.0:
; CHECK-NOLSE-O1-NEXT: LBB7_1: ; %atomicrmw.start ; CHECK-NOLSE-O1-NEXT: LBB9_1: ; %atomicrmw.start
; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NOLSE-O1-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NOLSE-O1-NEXT: ldxr x8, [x0] ; CHECK-NOLSE-O1-NEXT: ldxr x8, [x0]
; CHECK-NOLSE-O1-NEXT: orr x9, x8, #0x7 ; CHECK-NOLSE-O1-NEXT: orr x9, x8, #0x7
; CHECK-NOLSE-O1-NEXT: stxr w10, x9, [x0] ; CHECK-NOLSE-O1-NEXT: stxr w10, x9, [x0]
; CHECK-NOLSE-O1-NEXT: cbnz w10, LBB7_1 ; CHECK-NOLSE-O1-NEXT: cbnz w10, LBB9_1
; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end ; CHECK-NOLSE-O1-NEXT: ; %bb.2: ; %atomicrmw.end
; CHECK-NOLSE-O1-NEXT: mov x0, x8 ; CHECK-NOLSE-O1-NEXT: mov x0, x8
; CHECK-NOLSE-O1-NEXT: ret ; CHECK-NOLSE-O1-NEXT: ret
; ;
; CHECK-NOLSE-O0-LABEL: fetch_and_or_64: ; CHECK-NOLSE-O0-LABEL: fetch_and_or_64:
; CHECK-NOLSE-O0: ; %bb.0: ; CHECK-NOLSE-O0: ; %bb.0:
; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: sub sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x0, [sp, #16] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: ldr x8, [x0] ; CHECK-NOLSE-O0-NEXT: ldr x8, [x0]
; CHECK-NOLSE-O0-NEXT: str x8, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x8, [sp, #24] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: LBB7_1: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB9_1: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1 ; CHECK-NOLSE-O0-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NOLSE-O0-NEXT: ; Child Loop BB7_2 Depth 2 ; CHECK-NOLSE-O0-NEXT: ; Child Loop BB9_2 Depth 2
; CHECK-NOLSE-O0-NEXT: ldr x8, [sp, #24] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x8, [sp, #24] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x11, [sp, #16] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: orr x12, x8, #0x7 ; CHECK-NOLSE-O0-NEXT: orr x12, x8, #0x7
; CHECK-NOLSE-O0-NEXT: LBB7_2: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB9_2: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB7_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; Parent Loop BB9_1 Depth=1
; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2 ; CHECK-NOLSE-O0-NEXT: ; => This Inner Loop Header: Depth=2
; CHECK-NOLSE-O0-NEXT: ldaxr x9, [x11] ; CHECK-NOLSE-O0-NEXT: ldaxr x9, [x11]
; CHECK-NOLSE-O0-NEXT: cmp x9, x8 ; CHECK-NOLSE-O0-NEXT: cmp x9, x8
; CHECK-NOLSE-O0-NEXT: b.ne LBB7_4 ; CHECK-NOLSE-O0-NEXT: b.ne LBB9_4
; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: ; %bb.3: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB7_2 Depth=2 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB9_2 Depth=2
; CHECK-NOLSE-O0-NEXT: stlxr w10, x12, [x11] ; CHECK-NOLSE-O0-NEXT: stlxr w10, x12, [x11]
; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB7_2 ; CHECK-NOLSE-O0-NEXT: cbnz w10, LBB9_2
; CHECK-NOLSE-O0-NEXT: LBB7_4: ; %atomicrmw.start ; CHECK-NOLSE-O0-NEXT: LBB9_4: ; %atomicrmw.start
; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB7_1 Depth=1 ; CHECK-NOLSE-O0-NEXT: ; in Loop: Header=BB9_1 Depth=1
; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #8] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8 ; CHECK-NOLSE-O0-NEXT: subs x8, x9, x8
; CHECK-NOLSE-O0-NEXT: cset w8, eq ; CHECK-NOLSE-O0-NEXT: cset w8, eq
; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill ; CHECK-NOLSE-O0-NEXT: str x9, [sp, #24] ; 8-byte Folded Spill
; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB7_1 ; CHECK-NOLSE-O0-NEXT: tbz w8, #0, LBB9_1
; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end ; CHECK-NOLSE-O0-NEXT: ; %bb.5: ; %atomicrmw.end
; CHECK-NOLSE-O0-NEXT: ldr x0, [sp, #8] ; 8-byte Folded Reload ; CHECK-NOLSE-O0-NEXT: ldr x0, [sp, #8] ; 8-byte Folded Reload
; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32 ; CHECK-NOLSE-O0-NEXT: add sp, sp, #32 ; =32
; CHECK-NOLSE-O0-NEXT: ret ; CHECK-NOLSE-O0-NEXT: ret
; ;
; CHECK-LSE-O1-LABEL: fetch_and_or_64: ; CHECK-LSE-O1-LABEL: fetch_and_or_64:
; CHECK-LSE-O1: ; %bb.0: ; CHECK-LSE-O1: ; %bb.0:
; CHECK-LSE-O1-NEXT: mov w8, #7 ; CHECK-LSE-O1-NEXT: mov w8, #7
▲ Show 20 Lines • Show All 589 Lines • Show Last 20 Lines

llvm/test/CodeGen/AArch64/atomic-ops-lse.ll

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 388 Lines • ▼ Show 20 Lines
} }
define dso_local i8 @test_atomic_load_min_i8(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i8: ; CHECK-LABEL: test_atomic_load_min_i8:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB18_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB18_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i8* @var8, i8 %offset seq_cst %old = atomicrmw min i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_min_i16(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i16: ; CHECK-LABEL: test_atomic_load_min_i16:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB19_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB19_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i16* @var16, i16 %offset seq_cst %old = atomicrmw min i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_min_i32(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32: ; CHECK-LABEL: test_atomic_load_min_i32:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB20_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB20_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i32* @var32, i32 %offset seq_cst %old = atomicrmw min i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_min_i64(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64: ; CHECK-LABEL: test_atomic_load_min_i64:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB21_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB21_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i64* @var64, i64 %offset seq_cst %old = atomicrmw min i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_min_i32_noret(i32 %offset) nounwind { define dso_local void @test_atomic_load_min_i32_noret(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_noret: ; CHECK-LABEL: test_atomic_load_min_i32_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB22_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB22_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i32* @var32, i32 %offset seq_cst atomicrmw min i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_min_i64_noret(i64 %offset) nounwind { define dso_local void @test_atomic_load_min_i64_noret(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_noret: ; CHECK-LABEL: test_atomic_load_min_i64_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB23_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB23_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i64* @var64, i64 %offset seq_cst atomicrmw min i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i8: ; CHECK-LABEL: test_atomic_load_umin_i8:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB24_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB24_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i8* @var8, i8 %offset seq_cst %old = atomicrmw umin i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i16: ; CHECK-LABEL: test_atomic_load_umin_i16:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB25_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB25_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i16* @var16, i16 %offset seq_cst %old = atomicrmw umin i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32: ; CHECK-LABEL: test_atomic_load_umin_i32:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB26_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB26_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i32* @var32, i32 %offset seq_cst %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64: ; CHECK-LABEL: test_atomic_load_umin_i64:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB27_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB27_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i64* @var64, i64 %offset seq_cst %old = atomicrmw umin i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umin_i32_noret(i32 %offset) nounwind { define dso_local void @test_atomic_load_umin_i32_noret(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_noret: ; CHECK-LABEL: test_atomic_load_umin_i32_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB28_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB28_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i32* @var32, i32 %offset seq_cst atomicrmw umin i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umin_i64_noret(i64 %offset) nounwind { define dso_local void @test_atomic_load_umin_i64_noret(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_noret: ; CHECK-LABEL: test_atomic_load_umin_i64_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB29_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB29_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i64* @var64, i64 %offset seq_cst atomicrmw umin i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_max_i8(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i8: ; CHECK-LABEL: test_atomic_load_max_i8:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB30_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB30_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i8* @var8, i8 %offset seq_cst %old = atomicrmw max i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_max_i16(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i16: ; CHECK-LABEL: test_atomic_load_max_i16:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB31_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB31_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i16* @var16, i16 %offset seq_cst %old = atomicrmw max i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_max_i32(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32: ; CHECK-LABEL: test_atomic_load_max_i32:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB32_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB32_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i32* @var32, i32 %offset seq_cst %old = atomicrmw max i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_max_i64(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64: ; CHECK-LABEL: test_atomic_load_max_i64:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB33_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB33_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i64* @var64, i64 %offset seq_cst %old = atomicrmw max i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_max_i32_noret(i32 %offset) nounwind { define dso_local void @test_atomic_load_max_i32_noret(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_noret: ; CHECK-LABEL: test_atomic_load_max_i32_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB34_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB34_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i32* @var32, i32 %offset seq_cst atomicrmw max i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_max_i64_noret(i64 %offset) nounwind { define dso_local void @test_atomic_load_max_i64_noret(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_noret: ; CHECK-LABEL: test_atomic_load_max_i64_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB35_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB35_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i64* @var64, i64 %offset seq_cst atomicrmw max i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i8: ; CHECK-LABEL: test_atomic_load_umax_i8:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB36_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB36_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i8* @var8, i8 %offset seq_cst %old = atomicrmw umax i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i16: ; CHECK-LABEL: test_atomic_load_umax_i16:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB37_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB37_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i16* @var16, i16 %offset seq_cst %old = atomicrmw umax i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32: ; CHECK-LABEL: test_atomic_load_umax_i32:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB38_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB38_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i32* @var32, i32 %offset seq_cst %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64: ; CHECK-LABEL: test_atomic_load_umax_i64:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB39_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB39_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i64* @var64, i64 %offset seq_cst %old = atomicrmw umax i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umax_i32_noret(i32 %offset) nounwind { define dso_local void @test_atomic_load_umax_i32_noret(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_noret: ; CHECK-LABEL: test_atomic_load_umax_i32_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB40_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB40_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i32* @var32, i32 %offset seq_cst atomicrmw umax i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umax_i64_noret(i64 %offset) nounwind { define dso_local void @test_atomic_load_umax_i64_noret(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_noret: ; CHECK-LABEL: test_atomic_load_umax_i64_noret:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB41_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB41_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i64* @var64, i64 %offset seq_cst atomicrmw umax i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
▲ Show 20 Lines • Show All 244 Lines • ▼ Show 20 Lines
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: casa w0, w1, [x[[ADDR]]] ; CHECK: casa w0, w1, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i32 @test_atomic_cmpxchg_i32_monotonic_acquire(i32 %wanted, i32 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_monotonic_acquire:
; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x2, var32
; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq
; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; OUTLINE-ATOMICS-NEXT: ret
%pair = cmpxchg i32* @var32, i32 %wanted, i32 %new monotonic acquire
%old = extractvalue { i32, i1 } %pair, 0
; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: casa w0, w1, [x[[ADDR]]]
; CHECK-NOT: dmb
ret i32 %old
}
define dso_local i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { define dso_local i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i64: ; CHECK-LABEL: test_atomic_cmpxchg_i64:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64: ; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x2, var64 ; OUTLINE-ATOMICS-NEXT: adrp x2, var64
; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq ; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq
Show All 30 Lines
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
; CHECK: caspa x0, x1, x2, x3, [x[[ADDR]]] ; CHECK: caspa x0, x1, x2, x3, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i128 %old ret i128 %old
} }
define dso_local i128 @test_atomic_cmpxchg_i128_monotonic_seqcst(i128 %wanted, i128 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i128_monotonic_seqcst:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_monotonic_seqcst:
; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x4, var128
; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq_rel
; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; OUTLINE-ATOMICS-NEXT: ret
%pair = cmpxchg i128* @var128, i128 %wanted, i128 %new monotonic seq_cst
%old = extractvalue { i128, i1 } %pair, 0
; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var128
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
; CHECK: caspal x0, x1, x2, x3, [x[[ADDR]]]
; CHECK-NOT: dmb
ret i128 %old
}
define dso_local i128 @test_atomic_cmpxchg_i128_release_acquire(i128 %wanted, i128 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i128_release_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_release_acquire:
; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x4, var128
; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq_rel
; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; OUTLINE-ATOMICS-NEXT: ret
%pair = cmpxchg i128* @var128, i128 %wanted, i128 %new release acquire
%old = extractvalue { i128, i1 } %pair, 0
; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var128
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128
; CHECK: caspal x0, x1, x2, x3, [x[[ADDR]]]
; CHECK-NOT: dmb
ret i128 %old
}
define dso_local i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_sub_i8: ; CHECK-LABEL: test_atomic_load_sub_i8:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: neg w0, w0 ; OUTLINE-ATOMICS-NEXT: neg w0, w0
; OUTLINE-ATOMICS-NEXT: adrp x1, var8 ; OUTLINE-ATOMICS-NEXT: adrp x1, var8
; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8
▲ Show 20 Lines • Show All 2,177 Lines • ▼ Show 20 Lines
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: casal w0, w1, [x[[ADDR]]] ; CHECK: casal w0, w1, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i32 @test_atomic_cmpxchg_i32_monotonic_seq_cst(i32 %wanted, i32 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_monotonic_seq_cst:
; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x2, var32
; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq_rel
; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; OUTLINE-ATOMICS-NEXT: ret
%pair = cmpxchg i32* @var32, i32 %wanted, i32 %new monotonic seq_cst
%old = extractvalue { i32, i1 } %pair, 0
; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: casal w0, w1, [x[[ADDR]]]
; CHECK-NOT: dmb
ret i32 %old
}
define dso_local i32 @test_atomic_cmpxchg_i32_release_acquire(i32 %wanted, i32 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i32_release_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_release_acquire:
; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x2, var32
; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq_rel
; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; OUTLINE-ATOMICS-NEXT: ret
%pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release acquire
%old = extractvalue { i32, i1 } %pair, 0
; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: casal w0, w1, [x[[ADDR]]]
; CHECK-NOT: dmb
ret i32 %old
}
define dso_local i64 @test_atomic_cmpxchg_i64_seq_cst(i64 %wanted, i64 %new) nounwind { define dso_local i64 @test_atomic_cmpxchg_i64_seq_cst(i64 %wanted, i64 %new) nounwind {
; CHECK-LABEL: test_atomic_cmpxchg_i64_seq_cst: ; CHECK-LABEL: test_atomic_cmpxchg_i64_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; OUTLINE-ATOMICS-NEXT: adrp x2, var64 ; OUTLINE-ATOMICS-NEXT: adrp x2, var64
; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64
; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq_rel ; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq_rel
Show All 36 Lines
} }
define dso_local i8 @test_atomic_load_max_i8_acq_rel(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_max_i8_acq_rel(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i8_acq_rel: ; CHECK-LABEL: test_atomic_load_max_i8_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB158_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB158_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i8* @var8, i8 %offset acq_rel %old = atomicrmw max i8* @var8, i8 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_max_i16_acq_rel(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_max_i16_acq_rel(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i16_acq_rel: ; CHECK-LABEL: test_atomic_load_max_i16_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB159_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB159_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i16* @var16, i16 %offset acq_rel %old = atomicrmw max i16* @var16, i16 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_max_i32_acq_rel(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_max_i32_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_acq_rel: ; CHECK-LABEL: test_atomic_load_max_i32_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB160_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB160_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i32* @var32, i32 %offset acq_rel %old = atomicrmw max i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_max_i64_acq_rel(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_max_i64_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_acq_rel: ; CHECK-LABEL: test_atomic_load_max_i64_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB161_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB161_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i64* @var64, i64 %offset acq_rel %old = atomicrmw max i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_max_i32_noret_acq_rel(i32 %offset) nounwind { define dso_local void @test_atomic_load_max_i32_noret_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_max_i32_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB162_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB162_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i32* @var32, i32 %offset acq_rel atomicrmw max i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_max_i64_noret_acq_rel(i64 %offset) nounwind { define dso_local void @test_atomic_load_max_i64_noret_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_max_i64_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB163_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB163_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i64* @var64, i64 %offset acq_rel atomicrmw max i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_max_i8_acquire(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_max_i8_acquire(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i8_acquire: ; CHECK-LABEL: test_atomic_load_max_i8_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB164_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB164_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i8* @var8, i8 %offset acquire %old = atomicrmw max i8* @var8, i8 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsmaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_max_i16_acquire(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_max_i16_acquire(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i16_acquire: ; CHECK-LABEL: test_atomic_load_max_i16_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB165_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB165_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i16* @var16, i16 %offset acquire %old = atomicrmw max i16* @var16, i16 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsmaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_max_i32_acquire(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_max_i32_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_acquire: ; CHECK-LABEL: test_atomic_load_max_i32_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB166_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB166_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i32* @var32, i32 %offset acquire %old = atomicrmw max i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_max_i64_acquire(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_max_i64_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_acquire: ; CHECK-LABEL: test_atomic_load_max_i64_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB167_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB167_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i64* @var64, i64 %offset acquire %old = atomicrmw max i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_max_i32_noret_acquire(i32 %offset) nounwind { define dso_local void @test_atomic_load_max_i32_noret_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_noret_acquire: ; CHECK-LABEL: test_atomic_load_max_i32_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB168_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB168_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i32* @var32, i32 %offset acquire atomicrmw max i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_max_i64_noret_acquire(i64 %offset) nounwind { define dso_local void @test_atomic_load_max_i64_noret_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_noret_acquire: ; CHECK-LABEL: test_atomic_load_max_i64_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB169_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB169_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i64* @var64, i64 %offset acquire atomicrmw max i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_max_i8_monotonic(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_max_i8_monotonic(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i8_monotonic: ; CHECK-LABEL: test_atomic_load_max_i8_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB170_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB170_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i8* @var8, i8 %offset monotonic %old = atomicrmw max i8* @var8, i8 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsmaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_max_i16_monotonic(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_max_i16_monotonic(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i16_monotonic: ; CHECK-LABEL: test_atomic_load_max_i16_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB171_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB171_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i16* @var16, i16 %offset monotonic %old = atomicrmw max i16* @var16, i16 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsmaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_max_i32_monotonic(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_max_i32_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_monotonic: ; CHECK-LABEL: test_atomic_load_max_i32_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB172_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB172_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i32* @var32, i32 %offset monotonic %old = atomicrmw max i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_max_i64_monotonic(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_max_i64_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_monotonic: ; CHECK-LABEL: test_atomic_load_max_i64_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB173_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB173_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i64* @var64, i64 %offset monotonic %old = atomicrmw max i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_max_i32_noret_monotonic(i32 %offset) nounwind { define dso_local void @test_atomic_load_max_i32_noret_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_noret_monotonic: ; CHECK-LABEL: test_atomic_load_max_i32_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB174_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB174_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i32* @var32, i32 %offset monotonic atomicrmw max i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmax w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsmax w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_max_i64_noret_monotonic(i64 %offset) nounwind { define dso_local void @test_atomic_load_max_i64_noret_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_noret_monotonic: ; CHECK-LABEL: test_atomic_load_max_i64_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB175_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB175_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i64* @var64, i64 %offset monotonic atomicrmw max i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmax x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsmax x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_max_i8_release(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_max_i8_release(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i8_release: ; CHECK-LABEL: test_atomic_load_max_i8_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB176_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB176_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i8* @var8, i8 %offset release %old = atomicrmw max i8* @var8, i8 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsmaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_max_i16_release(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_max_i16_release(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i16_release: ; CHECK-LABEL: test_atomic_load_max_i16_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB177_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB177_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i16* @var16, i16 %offset release %old = atomicrmw max i16* @var16, i16 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsmaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_max_i32_release(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_max_i32_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_release: ; CHECK-LABEL: test_atomic_load_max_i32_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB178_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB178_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i32* @var32, i32 %offset release %old = atomicrmw max i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_max_i64_release(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_max_i64_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_release: ; CHECK-LABEL: test_atomic_load_max_i64_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB179_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB179_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i64* @var64, i64 %offset release %old = atomicrmw max i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_max_i32_noret_release(i32 %offset) nounwind { define dso_local void @test_atomic_load_max_i32_noret_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_noret_release: ; CHECK-LABEL: test_atomic_load_max_i32_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB180_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB180_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i32* @var32, i32 %offset release atomicrmw max i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsmaxl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_max_i64_noret_release(i64 %offset) nounwind { define dso_local void @test_atomic_load_max_i64_noret_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_noret_release: ; CHECK-LABEL: test_atomic_load_max_i64_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB181_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB181_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i64* @var64, i64 %offset release atomicrmw max i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsmaxl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_max_i8_seq_cst(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_max_i8_seq_cst(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i8_seq_cst: ; CHECK-LABEL: test_atomic_load_max_i8_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB182_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB182_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i8* @var8, i8 %offset seq_cst %old = atomicrmw max i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_max_i16_seq_cst(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_max_i16_seq_cst(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i16_seq_cst: ; CHECK-LABEL: test_atomic_load_max_i16_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB183_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB183_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i16* @var16, i16 %offset seq_cst %old = atomicrmw max i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_max_i32_seq_cst(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_max_i32_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_seq_cst: ; CHECK-LABEL: test_atomic_load_max_i32_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB184_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB184_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i32* @var32, i32 %offset seq_cst %old = atomicrmw max i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_max_i64_seq_cst(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_max_i64_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_seq_cst: ; CHECK-LABEL: test_atomic_load_max_i64_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB185_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB185_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw max i64* @var64, i64 %offset seq_cst %old = atomicrmw max i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_max_i32_noret_seq_cst(i32 %offset) nounwind { define dso_local void @test_atomic_load_max_i32_noret_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i32_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_max_i32_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB186_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB186_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i32* @var32, i32 %offset seq_cst atomicrmw max i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_max_i64_noret_seq_cst(i64 %offset) nounwind { define dso_local void @test_atomic_load_max_i64_noret_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_max_i64_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_max_i64_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB187_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB187_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw max i64* @var64, i64 %offset seq_cst atomicrmw max i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_min_i8_acq_rel(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_min_i8_acq_rel(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i8_acq_rel: ; CHECK-LABEL: test_atomic_load_min_i8_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB188_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB188_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i8* @var8, i8 %offset acq_rel %old = atomicrmw min i8* @var8, i8 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_min_i16_acq_rel(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_min_i16_acq_rel(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i16_acq_rel: ; CHECK-LABEL: test_atomic_load_min_i16_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB189_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB189_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i16* @var16, i16 %offset acq_rel %old = atomicrmw min i16* @var16, i16 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_min_i32_acq_rel(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_min_i32_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_acq_rel: ; CHECK-LABEL: test_atomic_load_min_i32_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB190_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB190_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i32* @var32, i32 %offset acq_rel %old = atomicrmw min i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_min_i64_acq_rel(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_min_i64_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_acq_rel: ; CHECK-LABEL: test_atomic_load_min_i64_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB191_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB191_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i64* @var64, i64 %offset acq_rel %old = atomicrmw min i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_min_i32_noret_acq_rel(i32 %offset) nounwind { define dso_local void @test_atomic_load_min_i32_noret_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_min_i32_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB192_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB192_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i32* @var32, i32 %offset acq_rel atomicrmw min i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_min_i64_noret_acq_rel(i64 %offset) nounwind { define dso_local void @test_atomic_load_min_i64_noret_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_min_i64_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB193_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB193_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i64* @var64, i64 %offset acq_rel atomicrmw min i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_min_i8_acquire(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_min_i8_acquire(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i8_acquire: ; CHECK-LABEL: test_atomic_load_min_i8_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB194_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB194_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i8* @var8, i8 %offset acquire %old = atomicrmw min i8* @var8, i8 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_min_i16_acquire(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_min_i16_acquire(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i16_acquire: ; CHECK-LABEL: test_atomic_load_min_i16_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB195_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB195_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i16* @var16, i16 %offset acquire %old = atomicrmw min i16* @var16, i16 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_min_i32_acquire(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_min_i32_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_acquire: ; CHECK-LABEL: test_atomic_load_min_i32_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB196_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB196_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i32* @var32, i32 %offset acquire %old = atomicrmw min i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_min_i64_acquire(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_min_i64_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_acquire: ; CHECK-LABEL: test_atomic_load_min_i64_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB197_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB197_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i64* @var64, i64 %offset acquire %old = atomicrmw min i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_min_i32_noret_acquire(i32 %offset) nounwind { define dso_local void @test_atomic_load_min_i32_noret_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_noret_acquire: ; CHECK-LABEL: test_atomic_load_min_i32_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB198_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB198_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i32* @var32, i32 %offset acquire atomicrmw min i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmina w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmina w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_min_i64_noret_acquire(i64 %offset) nounwind { define dso_local void @test_atomic_load_min_i64_noret_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_noret_acquire: ; CHECK-LABEL: test_atomic_load_min_i64_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB199_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB199_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i64* @var64, i64 %offset acquire atomicrmw min i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmina x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmina x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_min_i8_monotonic(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_min_i8_monotonic(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i8_monotonic: ; CHECK-LABEL: test_atomic_load_min_i8_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB200_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB200_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i8* @var8, i8 %offset monotonic %old = atomicrmw min i8* @var8, i8 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_min_i16_monotonic(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_min_i16_monotonic(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i16_monotonic: ; CHECK-LABEL: test_atomic_load_min_i16_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB201_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB201_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i16* @var16, i16 %offset monotonic %old = atomicrmw min i16* @var16, i16 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_min_i32_monotonic(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_min_i32_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_monotonic: ; CHECK-LABEL: test_atomic_load_min_i32_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB202_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB202_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i32* @var32, i32 %offset monotonic %old = atomicrmw min i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_min_i64_monotonic(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_min_i64_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_monotonic: ; CHECK-LABEL: test_atomic_load_min_i64_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB203_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB203_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i64* @var64, i64 %offset monotonic %old = atomicrmw min i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsmin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_min_i32_noret_monotonic(i32 %offset) nounwind { define dso_local void @test_atomic_load_min_i32_noret_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_noret_monotonic: ; CHECK-LABEL: test_atomic_load_min_i32_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB204_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB204_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i32* @var32, i32 %offset monotonic atomicrmw min i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsmin w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsmin w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_min_i64_noret_monotonic(i64 %offset) nounwind { define dso_local void @test_atomic_load_min_i64_noret_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_noret_monotonic: ; CHECK-LABEL: test_atomic_load_min_i64_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB205_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB205_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i64* @var64, i64 %offset monotonic atomicrmw min i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsmin x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsmin x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_min_i8_release(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_min_i8_release(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i8_release: ; CHECK-LABEL: test_atomic_load_min_i8_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB206_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB206_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i8* @var8, i8 %offset release %old = atomicrmw min i8* @var8, i8 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_min_i16_release(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_min_i16_release(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i16_release: ; CHECK-LABEL: test_atomic_load_min_i16_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB207_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB207_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i16* @var16, i16 %offset release %old = atomicrmw min i16* @var16, i16 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_min_i32_release(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_min_i32_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_release: ; CHECK-LABEL: test_atomic_load_min_i32_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB208_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB208_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i32* @var32, i32 %offset release %old = atomicrmw min i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_min_i64_release(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_min_i64_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_release: ; CHECK-LABEL: test_atomic_load_min_i64_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB209_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB209_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i64* @var64, i64 %offset release %old = atomicrmw min i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_min_i32_noret_release(i32 %offset) nounwind { define dso_local void @test_atomic_load_min_i32_noret_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_noret_release: ; CHECK-LABEL: test_atomic_load_min_i32_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB210_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB210_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i32* @var32, i32 %offset release atomicrmw min i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsminl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_min_i64_noret_release(i64 %offset) nounwind { define dso_local void @test_atomic_load_min_i64_noret_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_noret_release: ; CHECK-LABEL: test_atomic_load_min_i64_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB211_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB211_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i64* @var64, i64 %offset release atomicrmw min i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldsminl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_min_i8_seq_cst(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_min_i8_seq_cst(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i8_seq_cst: ; CHECK-LABEL: test_atomic_load_min_i8_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB212_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 ; OUTLINE-ATOMICS-NEXT: sxtb w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB212_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i8* @var8, i8 %offset seq_cst %old = atomicrmw min i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_min_i16_seq_cst(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_min_i16_seq_cst(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i16_seq_cst: ; CHECK-LABEL: test_atomic_load_min_i16_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB213_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9]
; OUTLINE-ATOMICS-NEXT: sxth w8, w10 ; OUTLINE-ATOMICS-NEXT: sxth w8, w10
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth
; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB213_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i16* @var16, i16 %offset seq_cst %old = atomicrmw min i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_min_i32_seq_cst(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_min_i32_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_seq_cst: ; CHECK-LABEL: test_atomic_load_min_i32_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB214_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB214_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i32* @var32, i32 %offset seq_cst %old = atomicrmw min i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_min_i64_seq_cst(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_min_i64_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_seq_cst: ; CHECK-LABEL: test_atomic_load_min_i64_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB215_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB215_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw min i64* @var64, i64 %offset seq_cst %old = atomicrmw min i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_min_i32_noret_seq_cst(i32 %offset) nounwind { define dso_local void @test_atomic_load_min_i32_noret_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i32_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_min_i32_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB216_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB216_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i32* @var32, i32 %offset seq_cst atomicrmw min i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_min_i64_noret_seq_cst(i64 %offset) nounwind { define dso_local void @test_atomic_load_min_i64_noret_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_min_i64_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_min_i64_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB217_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB217_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw min i64* @var64, i64 %offset seq_cst atomicrmw min i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
▲ Show 20 Lines • Show All 1,942 Lines • ▼ Show 20 Lines
} }
define dso_local i8 @test_atomic_load_umax_i8_acq_rel(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umax_i8_acq_rel(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i8_acq_rel: ; CHECK-LABEL: test_atomic_load_umax_i8_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB308_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB308_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i8* @var8, i8 %offset acq_rel %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umax_i16_acq_rel(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umax_i16_acq_rel(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i16_acq_rel: ; CHECK-LABEL: test_atomic_load_umax_i16_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB309_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB309_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i16* @var16, i16 %offset acq_rel %old = atomicrmw umax i16* @var16, i16 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umax_i32_acq_rel(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umax_i32_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_acq_rel: ; CHECK-LABEL: test_atomic_load_umax_i32_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB310_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB310_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i32* @var32, i32 %offset acq_rel %old = atomicrmw umax i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umax_i64_acq_rel(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umax_i64_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_acq_rel: ; CHECK-LABEL: test_atomic_load_umax_i64_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB311_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB311_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i64* @var64, i64 %offset acq_rel %old = atomicrmw umax i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umax_i32_noret_acq_rel(i32 %offset) nounwind { define dso_local void @test_atomic_load_umax_i32_noret_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_umax_i32_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB312_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB312_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i32* @var32, i32 %offset acq_rel atomicrmw umax i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umax_i64_noret_acq_rel(i64 %offset) nounwind { define dso_local void @test_atomic_load_umax_i64_noret_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_umax_i64_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB313_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB313_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i64* @var64, i64 %offset acq_rel atomicrmw umax i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umax_i8_acquire(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umax_i8_acquire(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i8_acquire: ; CHECK-LABEL: test_atomic_load_umax_i8_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB314_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB314_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i8* @var8, i8 %offset acquire %old = atomicrmw umax i8* @var8, i8 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldumaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umax_i16_acquire(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umax_i16_acquire(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i16_acquire: ; CHECK-LABEL: test_atomic_load_umax_i16_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB315_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB315_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i16* @var16, i16 %offset acquire %old = atomicrmw umax i16* @var16, i16 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldumaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umax_i32_acquire(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umax_i32_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_acquire: ; CHECK-LABEL: test_atomic_load_umax_i32_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB316_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB316_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i32* @var32, i32 %offset acquire %old = atomicrmw umax i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umax_i64_acquire(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umax_i64_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_acquire: ; CHECK-LABEL: test_atomic_load_umax_i64_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB317_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB317_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i64* @var64, i64 %offset acquire %old = atomicrmw umax i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umax_i32_noret_acquire(i32 %offset) nounwind { define dso_local void @test_atomic_load_umax_i32_noret_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_noret_acquire: ; CHECK-LABEL: test_atomic_load_umax_i32_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB318_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB318_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i32* @var32, i32 %offset acquire atomicrmw umax i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umax_i64_noret_acquire(i64 %offset) nounwind { define dso_local void @test_atomic_load_umax_i64_noret_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_noret_acquire: ; CHECK-LABEL: test_atomic_load_umax_i64_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB319_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB319_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i64* @var64, i64 %offset acquire atomicrmw umax i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umax_i8_monotonic(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umax_i8_monotonic(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i8_monotonic: ; CHECK-LABEL: test_atomic_load_umax_i8_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB320_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB320_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i8* @var8, i8 %offset monotonic %old = atomicrmw umax i8* @var8, i8 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldumaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umax_i16_monotonic(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umax_i16_monotonic(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i16_monotonic: ; CHECK-LABEL: test_atomic_load_umax_i16_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB321_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB321_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i16* @var16, i16 %offset monotonic %old = atomicrmw umax i16* @var16, i16 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldumaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umax_i32_monotonic(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umax_i32_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_monotonic: ; CHECK-LABEL: test_atomic_load_umax_i32_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB322_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB322_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i32* @var32, i32 %offset monotonic %old = atomicrmw umax i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umax_i64_monotonic(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umax_i64_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_monotonic: ; CHECK-LABEL: test_atomic_load_umax_i64_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB323_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB323_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i64* @var64, i64 %offset monotonic %old = atomicrmw umax i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umax_i32_noret_monotonic(i32 %offset) nounwind { define dso_local void @test_atomic_load_umax_i32_noret_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_noret_monotonic: ; CHECK-LABEL: test_atomic_load_umax_i32_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB324_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB324_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i32* @var32, i32 %offset monotonic atomicrmw umax i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumax w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldumax w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umax_i64_noret_monotonic(i64 %offset) nounwind { define dso_local void @test_atomic_load_umax_i64_noret_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_noret_monotonic: ; CHECK-LABEL: test_atomic_load_umax_i64_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB325_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB325_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i64* @var64, i64 %offset monotonic atomicrmw umax i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumax x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldumax x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umax_i8_release(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umax_i8_release(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i8_release: ; CHECK-LABEL: test_atomic_load_umax_i8_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB326_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB326_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i8* @var8, i8 %offset release %old = atomicrmw umax i8* @var8, i8 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldumaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umax_i16_release(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umax_i16_release(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i16_release: ; CHECK-LABEL: test_atomic_load_umax_i16_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB327_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB327_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i16* @var16, i16 %offset release %old = atomicrmw umax i16* @var16, i16 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldumaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umax_i32_release(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umax_i32_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_release: ; CHECK-LABEL: test_atomic_load_umax_i32_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB328_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB328_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i32* @var32, i32 %offset release %old = atomicrmw umax i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umax_i64_release(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umax_i64_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_release: ; CHECK-LABEL: test_atomic_load_umax_i64_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB329_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB329_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i64* @var64, i64 %offset release %old = atomicrmw umax i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umax_i32_noret_release(i32 %offset) nounwind { define dso_local void @test_atomic_load_umax_i32_noret_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_noret_release: ; CHECK-LABEL: test_atomic_load_umax_i32_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB330_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB330_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i32* @var32, i32 %offset release atomicrmw umax i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldumaxl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umax_i64_noret_release(i64 %offset) nounwind { define dso_local void @test_atomic_load_umax_i64_noret_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_noret_release: ; CHECK-LABEL: test_atomic_load_umax_i64_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB331_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB331_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i64* @var64, i64 %offset release atomicrmw umax i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldumaxl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umax_i8_seq_cst(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umax_i8_seq_cst(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i8_seq_cst: ; CHECK-LABEL: test_atomic_load_umax_i8_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB332_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB332_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i8* @var8, i8 %offset seq_cst %old = atomicrmw umax i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umax_i16_seq_cst(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umax_i16_seq_cst(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i16_seq_cst: ; CHECK-LABEL: test_atomic_load_umax_i16_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB333_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB333_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i16* @var16, i16 %offset seq_cst %old = atomicrmw umax i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umax_i32_seq_cst(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umax_i32_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_seq_cst: ; CHECK-LABEL: test_atomic_load_umax_i32_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB334_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB334_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i32* @var32, i32 %offset seq_cst %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umax_i64_seq_cst(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umax_i64_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_seq_cst: ; CHECK-LABEL: test_atomic_load_umax_i64_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB335_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB335_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umax i64* @var64, i64 %offset seq_cst %old = atomicrmw umax i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umax_i32_noret_seq_cst(i32 %offset) nounwind { define dso_local void @test_atomic_load_umax_i32_noret_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i32_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_umax_i32_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB336_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB336_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i32* @var32, i32 %offset seq_cst atomicrmw umax i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umax_i64_noret_seq_cst(i64 %offset) nounwind { define dso_local void @test_atomic_load_umax_i64_noret_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umax_i64_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_umax_i64_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB337_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB337_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umax i64* @var64, i64 %offset seq_cst atomicrmw umax i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umin_i8_acq_rel(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umin_i8_acq_rel(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i8_acq_rel: ; CHECK-LABEL: test_atomic_load_umin_i8_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB338_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB338_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i8* @var8, i8 %offset acq_rel %old = atomicrmw umin i8* @var8, i8 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umin_i16_acq_rel(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umin_i16_acq_rel(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i16_acq_rel: ; CHECK-LABEL: test_atomic_load_umin_i16_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB339_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB339_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i16* @var16, i16 %offset acq_rel %old = atomicrmw umin i16* @var16, i16 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umin_i32_acq_rel(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umin_i32_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_acq_rel: ; CHECK-LABEL: test_atomic_load_umin_i32_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB340_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB340_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i32* @var32, i32 %offset acq_rel %old = atomicrmw umin i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umin_i64_acq_rel(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umin_i64_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_acq_rel: ; CHECK-LABEL: test_atomic_load_umin_i64_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB341_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB341_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i64* @var64, i64 %offset acq_rel %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umin_i32_noret_acq_rel(i32 %offset) nounwind { define dso_local void @test_atomic_load_umin_i32_noret_acq_rel(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_umin_i32_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB342_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB342_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i32* @var32, i32 %offset acq_rel atomicrmw umin i32* @var32, i32 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umin_i64_noret_acq_rel(i64 %offset) nounwind { define dso_local void @test_atomic_load_umin_i64_noret_acq_rel(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_noret_acq_rel: ; CHECK-LABEL: test_atomic_load_umin_i64_noret_acq_rel:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acq_rel: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acq_rel:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB343_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB343_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i64* @var64, i64 %offset acq_rel atomicrmw umin i64* @var64, i64 %offset acq_rel
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umin_i8_acquire(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umin_i8_acquire(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i8_acquire: ; CHECK-LABEL: test_atomic_load_umin_i8_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB344_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB344_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i8* @var8, i8 %offset acquire %old = atomicrmw umin i8* @var8, i8 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: lduminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umin_i16_acquire(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umin_i16_acquire(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i16_acquire: ; CHECK-LABEL: test_atomic_load_umin_i16_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB345_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB345_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i16* @var16, i16 %offset acquire %old = atomicrmw umin i16* @var16, i16 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: lduminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umin_i32_acquire(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umin_i32_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_acquire: ; CHECK-LABEL: test_atomic_load_umin_i32_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB346_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB346_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i32* @var32, i32 %offset acquire %old = atomicrmw umin i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umin_i64_acquire(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umin_i64_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_acquire: ; CHECK-LABEL: test_atomic_load_umin_i64_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB347_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB347_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i64* @var64, i64 %offset acquire %old = atomicrmw umin i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umin_i32_noret_acquire(i32 %offset) nounwind { define dso_local void @test_atomic_load_umin_i32_noret_acquire(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_noret_acquire: ; CHECK-LABEL: test_atomic_load_umin_i32_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB348_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB348_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i32* @var32, i32 %offset acquire atomicrmw umin i32* @var32, i32 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumina w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumina w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umin_i64_noret_acquire(i64 %offset) nounwind { define dso_local void @test_atomic_load_umin_i64_noret_acquire(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_noret_acquire: ; CHECK-LABEL: test_atomic_load_umin_i64_noret_acquire:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acquire: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acquire:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB349_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB349_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i64* @var64, i64 %offset acquire atomicrmw umin i64* @var64, i64 %offset acquire
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumina x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumina x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umin_i8_monotonic(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umin_i8_monotonic(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i8_monotonic: ; CHECK-LABEL: test_atomic_load_umin_i8_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB350_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB350_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i8* @var8, i8 %offset monotonic %old = atomicrmw umin i8* @var8, i8 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: lduminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umin_i16_monotonic(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umin_i16_monotonic(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i16_monotonic: ; CHECK-LABEL: test_atomic_load_umin_i16_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB351_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB351_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i16* @var16, i16 %offset monotonic %old = atomicrmw umin i16* @var16, i16 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: lduminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umin_i32_monotonic(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umin_i32_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_monotonic: ; CHECK-LABEL: test_atomic_load_umin_i32_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB352_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB352_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i32* @var32, i32 %offset monotonic %old = atomicrmw umin i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umin_i64_monotonic(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umin_i64_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_monotonic: ; CHECK-LABEL: test_atomic_load_umin_i64_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB353_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls
; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB353_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i64* @var64, i64 %offset monotonic %old = atomicrmw umin i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: ldumin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umin_i32_noret_monotonic(i32 %offset) nounwind { define dso_local void @test_atomic_load_umin_i32_noret_monotonic(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_noret_monotonic: ; CHECK-LABEL: test_atomic_load_umin_i32_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB354_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls
; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB354_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i32* @var32, i32 %offset monotonic atomicrmw umin i32* @var32, i32 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: ldumin w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldumin w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umin_i64_noret_monotonic(i64 %offset) nounwind { define dso_local void @test_atomic_load_umin_i64_noret_monotonic(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_noret_monotonic: ; CHECK-LABEL: test_atomic_load_umin_i64_noret_monotonic:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_monotonic: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_monotonic:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB355_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls
; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB355_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i64* @var64, i64 %offset monotonic atomicrmw umin i64* @var64, i64 %offset monotonic
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: ldumin x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: ldumin x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umin_i8_release(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umin_i8_release(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i8_release: ; CHECK-LABEL: test_atomic_load_umin_i8_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB356_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB356_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i8* @var8, i8 %offset release %old = atomicrmw umin i8* @var8, i8 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: lduminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umin_i16_release(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umin_i16_release(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i16_release: ; CHECK-LABEL: test_atomic_load_umin_i16_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB357_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB357_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i16* @var16, i16 %offset release %old = atomicrmw umin i16* @var16, i16 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: lduminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umin_i32_release(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umin_i32_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_release: ; CHECK-LABEL: test_atomic_load_umin_i32_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB358_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB358_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i32* @var32, i32 %offset release %old = atomicrmw umin i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umin_i64_release(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umin_i64_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_release: ; CHECK-LABEL: test_atomic_load_umin_i64_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB359_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB359_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i64* @var64, i64 %offset release %old = atomicrmw umin i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umin_i32_noret_release(i32 %offset) nounwind { define dso_local void @test_atomic_load_umin_i32_noret_release(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_noret_release: ; CHECK-LABEL: test_atomic_load_umin_i32_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB360_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB360_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i32* @var32, i32 %offset release atomicrmw umin i32* @var32, i32 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: lduminl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umin_i64_noret_release(i64 %offset) nounwind { define dso_local void @test_atomic_load_umin_i64_noret_release(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_noret_release: ; CHECK-LABEL: test_atomic_load_umin_i64_noret_release:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_release: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_release:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB361_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB361_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i64* @var64, i64 %offset release atomicrmw umin i64* @var64, i64 %offset release
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] ; CHECK: lduminl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local i8 @test_atomic_load_umin_i8_seq_cst(i8 %offset) nounwind { define dso_local i8 @test_atomic_load_umin_i8_seq_cst(i8 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i8_seq_cst: ; CHECK-LABEL: test_atomic_load_umin_i8_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var8 ; OUTLINE-ATOMICS-NEXT: adrp x9, var8
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8
; OUTLINE-ATOMICS-NEXT: .LBB362_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB362_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i8* @var8, i8 %offset seq_cst %old = atomicrmw umin i8* @var8, i8 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i8 %old ret i8 %old
} }
define dso_local i16 @test_atomic_load_umin_i16_seq_cst(i16 %offset) nounwind { define dso_local i16 @test_atomic_load_umin_i16_seq_cst(i16 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i16_seq_cst: ; CHECK-LABEL: test_atomic_load_umin_i16_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var16 ; OUTLINE-ATOMICS-NEXT: adrp x9, var16
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16
; OUTLINE-ATOMICS-NEXT: .LBB363_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth ; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB363_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i16* @var16, i16 %offset seq_cst %old = atomicrmw umin i16* @var16, i16 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i16 %old ret i16 %old
} }
define dso_local i32 @test_atomic_load_umin_i32_seq_cst(i32 %offset) nounwind { define dso_local i32 @test_atomic_load_umin_i32_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_seq_cst: ; CHECK-LABEL: test_atomic_load_umin_i32_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var32 ; OUTLINE-ATOMICS-NEXT: adrp x9, var32
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB364_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp w8, w0 ; OUTLINE-ATOMICS-NEXT: cmp w8, w0
; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB364_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov w0, w8 ; OUTLINE-ATOMICS-NEXT: mov w0, w8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i32* @var32, i32 %offset seq_cst %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i32 %old ret i32 %old
} }
define dso_local i64 @test_atomic_load_umin_i64_seq_cst(i64 %offset) nounwind { define dso_local i64 @test_atomic_load_umin_i64_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_seq_cst: ; CHECK-LABEL: test_atomic_load_umin_i64_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x9, var64 ; OUTLINE-ATOMICS-NEXT: adrp x9, var64
; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB365_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] ; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9]
; OUTLINE-ATOMICS-NEXT: cmp x8, x0 ; OUTLINE-ATOMICS-NEXT: cmp x8, x0
; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] ; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9]
; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB365_1 ; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: mov x0, x8 ; OUTLINE-ATOMICS-NEXT: mov x0, x8
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
%old = atomicrmw umin i64* @var64, i64 %offset seq_cst %old = atomicrmw umin i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret i64 %old ret i64 %old
} }
define dso_local void @test_atomic_load_umin_i32_noret_seq_cst(i32 %offset) nounwind { define dso_local void @test_atomic_load_umin_i32_noret_seq_cst(i32 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i32_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_umin_i32_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var32 ; OUTLINE-ATOMICS-NEXT: adrp x8, var32
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32
; OUTLINE-ATOMICS-NEXT: .LBB366_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp w9, w0 ; OUTLINE-ATOMICS-NEXT: cmp w9, w0
; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls ; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB366_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i32* @var32, i32 %offset seq_cst atomicrmw umin i32* @var32, i32 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]]
; CHECK-NOT: dmb ; CHECK-NOT: dmb
ret void ret void
} }
define dso_local void @test_atomic_load_umin_i64_noret_seq_cst(i64 %offset) nounwind { define dso_local void @test_atomic_load_umin_i64_noret_seq_cst(i64 %offset) nounwind {
; CHECK-LABEL: test_atomic_load_umin_i64_noret_seq_cst: ; CHECK-LABEL: test_atomic_load_umin_i64_noret_seq_cst:
; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_seq_cst: ; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_seq_cst:
; OUTLINE-ATOMICS: // %bb.0: ; OUTLINE-ATOMICS: // %bb.0:
; OUTLINE-ATOMICS-NEXT: adrp x8, var64 ; OUTLINE-ATOMICS-NEXT: adrp x8, var64
; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 ; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64
; OUTLINE-ATOMICS-NEXT: .LBB367_1: // %atomicrmw.start ; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start
; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 ; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] ; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8]
; OUTLINE-ATOMICS-NEXT: cmp x9, x0 ; OUTLINE-ATOMICS-NEXT: cmp x9, x0
; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls ; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls
; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] ; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8]
; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB367_1 ; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]]
; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end ; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
; OUTLINE-ATOMICS-NEXT: ret ; OUTLINE-ATOMICS-NEXT: ret
atomicrmw umin i64* @var64, i64 %offset seq_cst atomicrmw umin i64* @var64, i64 %offset seq_cst
; CHECK-NOT: dmb ; CHECK-NOT: dmb
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] ; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]]
▲ Show 20 Lines • Show All 625 Lines • Show Last 20 Lines

llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll

Show First 20 Lines • Show All 1,771 Lines • ▼ Show 20 Lines
; RV32I-NEXT: call __atomic_compare_exchange_4@plt ; RV32I-NEXT: call __atomic_compare_exchange_4@plt
; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 16 ; RV32I-NEXT: addi sp, sp, 16
; RV32I-NEXT: ret ; RV32I-NEXT: ret
; ;
; RV32IA-LABEL: cmpxchg_i32_release_acquire: ; RV32IA-LABEL: cmpxchg_i32_release_acquire:
; RV32IA: # %bb.0: ; RV32IA: # %bb.0:
; RV32IA-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1 ; RV32IA-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1
; RV32IA-NEXT: lr.w a3, (a0) ; RV32IA-NEXT: lr.w.aq a3, (a0)
; RV32IA-NEXT: bne a3, a1, .LBB24_3 ; RV32IA-NEXT: bne a3, a1, .LBB24_3
; RV32IA-NEXT: # %bb.2: # in Loop: Header=BB24_1 Depth=1 ; RV32IA-NEXT: # %bb.2: # in Loop: Header=BB24_1 Depth=1
; RV32IA-NEXT: sc.w.rl a4, a2, (a0) ; RV32IA-NEXT: sc.w.rl a4, a2, (a0)
; RV32IA-NEXT: bnez a4, .LBB24_1 ; RV32IA-NEXT: bnez a4, .LBB24_1
; RV32IA-NEXT: .LBB24_3: ; RV32IA-NEXT: .LBB24_3:
; RV32IA-NEXT: ret ; RV32IA-NEXT: ret
; ;
; RV64I-LABEL: cmpxchg_i32_release_acquire: ; RV64I-LABEL: cmpxchg_i32_release_acquire:
; RV64I: # %bb.0: ; RV64I: # %bb.0:
; RV64I-NEXT: addi sp, sp, -16 ; RV64I-NEXT: addi sp, sp, -16
; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill ; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64I-NEXT: sw a1, 4(sp) ; RV64I-NEXT: sw a1, 4(sp)
; RV64I-NEXT: addi a1, sp, 4 ; RV64I-NEXT: addi a1, sp, 4
; RV64I-NEXT: addi a3, zero, 3 ; RV64I-NEXT: addi a3, zero, 3
; RV64I-NEXT: addi a4, zero, 2 ; RV64I-NEXT: addi a4, zero, 2
; RV64I-NEXT: call __atomic_compare_exchange_4@plt ; RV64I-NEXT: call __atomic_compare_exchange_4@plt
; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
; RV64I-NEXT: addi sp, sp, 16 ; RV64I-NEXT: addi sp, sp, 16
; RV64I-NEXT: ret ; RV64I-NEXT: ret
; ;
; RV64IA-LABEL: cmpxchg_i32_release_acquire: ; RV64IA-LABEL: cmpxchg_i32_release_acquire:
; RV64IA: # %bb.0: ; RV64IA: # %bb.0:
; RV64IA-NEXT: sext.w a1, a1 ; RV64IA-NEXT: sext.w a1, a1
; RV64IA-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1 ; RV64IA-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1
; RV64IA-NEXT: lr.w a3, (a0) ; RV64IA-NEXT: lr.w.aq a3, (a0)
; RV64IA-NEXT: bne a3, a1, .LBB24_3 ; RV64IA-NEXT: bne a3, a1, .LBB24_3
; RV64IA-NEXT: # %bb.2: # in Loop: Header=BB24_1 Depth=1 ; RV64IA-NEXT: # %bb.2: # in Loop: Header=BB24_1 Depth=1
; RV64IA-NEXT: sc.w.rl a4, a2, (a0) ; RV64IA-NEXT: sc.w.rl a4, a2, (a0)
; RV64IA-NEXT: bnez a4, .LBB24_1 ; RV64IA-NEXT: bnez a4, .LBB24_1
; RV64IA-NEXT: .LBB24_3: ; RV64IA-NEXT: .LBB24_3:
; RV64IA-NEXT: ret ; RV64IA-NEXT: ret
%res = cmpxchg i32* %ptr, i32 %cmp, i32 %val release acquire %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val release acquire
ret void ret void
▲ Show 20 Lines • Show All 556 Lines • ▼ Show 20 Lines
; RV64I-NEXT: call __atomic_compare_exchange_8@plt ; RV64I-NEXT: call __atomic_compare_exchange_8@plt
; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
; RV64I-NEXT: addi sp, sp, 16 ; RV64I-NEXT: addi sp, sp, 16
; RV64I-NEXT: ret ; RV64I-NEXT: ret
; ;
; RV64IA-LABEL: cmpxchg_i64_release_acquire: ; RV64IA-LABEL: cmpxchg_i64_release_acquire:
; RV64IA: # %bb.0: ; RV64IA: # %bb.0:
; RV64IA-NEXT: .LBB34_1: # =>This Inner Loop Header: Depth=1 ; RV64IA-NEXT: .LBB34_1: # =>This Inner Loop Header: Depth=1
; RV64IA-NEXT: lr.d a3, (a0) ; RV64IA-NEXT: lr.d.aq a3, (a0)
; RV64IA-NEXT: bne a3, a1, .LBB34_3 ; RV64IA-NEXT: bne a3, a1, .LBB34_3
; RV64IA-NEXT: # %bb.2: # in Loop: Header=BB34_1 Depth=1 ; RV64IA-NEXT: # %bb.2: # in Loop: Header=BB34_1 Depth=1
; RV64IA-NEXT: sc.d.rl a4, a2, (a0) ; RV64IA-NEXT: sc.d.rl a4, a2, (a0)
; RV64IA-NEXT: bnez a4, .LBB34_1 ; RV64IA-NEXT: bnez a4, .LBB34_1
; RV64IA-NEXT: .LBB34_3: ; RV64IA-NEXT: .LBB34_3:
; RV64IA-NEXT: ret ; RV64IA-NEXT: ret
%res = cmpxchg i64* %ptr, i64 %cmp, i64 %val release acquire %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val release acquire
ret void ret void
▲ Show 20 Lines • Show All 311 Lines • Show Last 20 Lines

llvm/utils/TableGen/CodeGenDAGPatterns.cpp

Show First 20 Lines • Show All 1,029 Lines • ▼ Show 20 Lines if (isLoad() || isStore() || isAtomic()) {
if (MemoryVT) if (MemoryVT)
Code += ("if (cast<MemSDNode>(N)->getMemoryVT() != MVT::" + Code += ("if (cast<MemSDNode>(N)->getMemoryVT() != MVT::" +
MemoryVT->getName() + ") return false;\n") MemoryVT->getName() + ") return false;\n")
.str(); .str();
} }
if (isAtomic() && isAtomicOrderingMonotonic()) if (isAtomic() && isAtomicOrderingMonotonic())
Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
"AtomicOrdering::Monotonic) return false;\n"; "AtomicOrdering::Monotonic) return false;\n";
if (isAtomic() && isAtomicOrderingAcquire()) if (isAtomic() && isAtomicOrderingAcquire())
Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
"AtomicOrdering::Acquire) return false;\n"; "AtomicOrdering::Acquire) return false;\n";
if (isAtomic() && isAtomicOrderingRelease()) if (isAtomic() && isAtomicOrderingRelease())
Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
"AtomicOrdering::Release) return false;\n"; "AtomicOrdering::Release) return false;\n";
if (isAtomic() && isAtomicOrderingAcquireRelease()) if (isAtomic() && isAtomicOrderingAcquireRelease())
Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
"AtomicOrdering::AcquireRelease) return false;\n"; "AtomicOrdering::AcquireRelease) return false;\n";
if (isAtomic() && isAtomicOrderingSequentiallyConsistent()) if (isAtomic() && isAtomicOrderingSequentiallyConsistent())
Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
"AtomicOrdering::SequentiallyConsistent) return false;\n"; "AtomicOrdering::SequentiallyConsistent) return false;\n";
if (isAtomic() && isAtomicOrderingAcquireOrStronger()) if (isAtomic() && isAtomicOrderingAcquireOrStronger())
Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
"return false;\n"; "return false;\n";
if (isAtomic() && isAtomicOrderingWeakerThanAcquire()) if (isAtomic() && isAtomicOrderingWeakerThanAcquire())
Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
"return false;\n"; "return false;\n";
if (isAtomic() && isAtomicOrderingReleaseOrStronger()) if (isAtomic() && isAtomicOrderingReleaseOrStronger())
Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
"return false;\n"; "return false;\n";
if (isAtomic() && isAtomicOrderingWeakerThanRelease()) if (isAtomic() && isAtomicOrderingWeakerThanRelease())
Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
"return false;\n"; "return false;\n";
if (isLoad() || isStore()) { if (isLoad() || isStore()) {
StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode"; StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode";
if (isUnindexed()) if (isUnindexed())
Code += ("if (cast<" + SDNodeName + Code += ("if (cast<" + SDNodeName +
">(N)->getAddressingMode() != ISD::UNINDEXED) " ">(N)->getAddressingMode() != ISD::UNINDEXED) "
▲ Show 20 Lines • Show All 3,654 Lines • Show Last 20 Lines