[PowerPC] Set setMaxAtomicSizeInBitsSupported appropriately for 32-bit PowerPC in PPCTargetLowering

[brad]

Summary

Set setMaxAtomicSizeInBitsSupported appropriately for 32-bit PowerPC in PPCTargetLowering.

[brad]

Drop the tests I had copied and modified from X86 as it looks like the existing tests are probably good enough and update the failing tests with update_llc_test_checks.py.

[brad]

Nothing changed. Just want to rebuild as an unrelated Windows test was broken.

[brad]

Any chance of a review of the auto generated tests. Do they look appropriate?

[nemanjai]

LGTM. Sorry for the delay.

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

 
   if (!isPPC64) {
     // These libcalls are not available in 32-bit.
     setLibcallName(RTLIB::SHL_I128, nullptr);
     setLibcallName(RTLIB::SRL_I128, nullptr);
     setLibcallName(RTLIB::SRA_I128, nullptr);
   }
 
+  if (!isPPC64)
+    setMaxAtomicSizeInBitsSupported(32);
+
   setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
 
   // We have target-specific dag combine patterns for the following nodes:
   setTargetDAGCombine(ISD::ADD);
   setTargetDAGCombine(ISD::SHL);
   setTargetDAGCombine(ISD::SRA);
   setTargetDAGCombine(ISD::SRL);
   setTargetDAGCombine(ISD::MUL);

llvm/test/CodeGen/PowerPC/atomic64.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O0 -mtriple=powerpc64-- -verify-machineinstrs | FileCheck %s --check-prefix POWERPC64
+; RUN: llc < %s -O0 -mtriple=powerpc-- -verify-machineinstrs | FileCheck %s --check-prefix POWERPC
+
+@sc64 = external global i64
+@fsc64 = external global double
+
+define void @atomic_fetch_add64() nounwind {
+; POWERPC64-LABEL: atomic_fetch_add64:
+; POWERPC64:       # %bb.0: # %entry
+; POWERPC64-NEXT:    addis 3, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; POWERPC64-NEXT:    li 4, 1
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB0_1: # %entry
+; POWERPC64-NEXT:    #
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    add 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB0_1
+; POWERPC64-NEXT:  # %bb.2: # %entry
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 3
+; POWERPC64-NEXT:    std 3, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB0_3: # %entry
+; POWERPC64-NEXT:    #
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    add 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB0_3
+; POWERPC64-NEXT:  # %bb.4: # %entry
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 5
+; POWERPC64-NEXT:    std 3, -32(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB0_5: # %entry
+; POWERPC64-NEXT:    #
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -32(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    add 6, 5, 4
+; POWERPC64-NEXT:    stdcx. 6, 0, 3
+; POWERPC64-NEXT:    std 4, -40(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    bne 0, .LBB0_5
+; POWERPC64-NEXT:  # %bb.6: # %entry
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:  .LBB0_7: # %entry
+; POWERPC64-NEXT:    #
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -40(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    add 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB0_7
+; POWERPC64-NEXT:  # %bb.8: # %entry
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_add64:
+; POWERPC:       # %bb.0: # %entry
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -48(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    li 4, 0
+; POWERPC-NEXT:    li 6, 1
+; POWERPC-NEXT:    li 5, 2
+; POWERPC-NEXT:    stw 3, 44(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 40(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 36(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_add_8
+; POWERPC-NEXT:    li 6, 3
+; POWERPC-NEXT:    lwz 5, 44(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 32(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_add_8
+; POWERPC-NEXT:    li 6, 5
+; POWERPC-NEXT:    lwz 5, 44(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_add_8
+; POWERPC-NEXT:    lwz 5, 44(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 16(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 16(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_fetch_add_8
+; POWERPC-NEXT:    lwz 0, 52(1)
+; POWERPC-NEXT:    addi 1, 1, 48
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+entry:
+  %t1 = atomicrmw add  i64* @sc64, i64 1 acquire
+  %t2 = atomicrmw add  i64* @sc64, i64 3 acquire
+  %t3 = atomicrmw add  i64* @sc64, i64 5 acquire
+  %t4 = atomicrmw add  i64* @sc64, i64 %t3 acquire
+  ret void
+}
+
+define void @atomic_fetch_sub64() nounwind {
+; POWERPC64-LABEL: atomic_fetch_sub64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 3, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; POWERPC64-NEXT:    li 4, 1
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB1_1:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    sub 4, 4, 5
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB1_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 3
+; POWERPC64-NEXT:    std 3, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB1_3:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    sub 4, 4, 5
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB1_3
+; POWERPC64-NEXT:  # %bb.4:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 5
+; POWERPC64-NEXT:    std 3, -32(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB1_5:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -32(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    sub 6, 4, 5
+; POWERPC64-NEXT:    stdcx. 6, 0, 3
+; POWERPC64-NEXT:    std 4, -40(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    bne 0, .LBB1_5
+; POWERPC64-NEXT:  # %bb.6:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:  .LBB1_7:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -40(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    sub 4, 4, 5
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB1_7
+; POWERPC64-NEXT:  # %bb.8:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_sub64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -48(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    li 4, 0
+; POWERPC-NEXT:    li 6, 1
+; POWERPC-NEXT:    li 5, 2
+; POWERPC-NEXT:    stw 3, 44(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 40(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 36(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_sub_8
+; POWERPC-NEXT:    li 6, 3
+; POWERPC-NEXT:    lwz 5, 44(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 32(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_sub_8
+; POWERPC-NEXT:    li 6, 5
+; POWERPC-NEXT:    lwz 5, 44(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_sub_8
+; POWERPC-NEXT:    lwz 5, 44(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 16(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 16(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    lwz 7, 40(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_fetch_sub_8
+; POWERPC-NEXT:    lwz 0, 52(1)
+; POWERPC-NEXT:    addi 1, 1, 48
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw sub  i64* @sc64, i64 1 acquire
+  %t2 = atomicrmw sub  i64* @sc64, i64 3 acquire
+  %t3 = atomicrmw sub  i64* @sc64, i64 5 acquire
+  %t4 = atomicrmw sub  i64* @sc64, i64 %t3 acquire
+  ret void
+}
+
+define void @atomic_fetch_and64() nounwind {
+; POWERPC64-LABEL: atomic_fetch_and64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 3, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; POWERPC64-NEXT:    li 4, 3
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB2_1:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    and 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB2_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 5
+; POWERPC64-NEXT:    std 3, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB2_3:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    and 6, 5, 4
+; POWERPC64-NEXT:    stdcx. 6, 0, 3
+; POWERPC64-NEXT:    std 4, -32(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    bne 0, .LBB2_3
+; POWERPC64-NEXT:  # %bb.4:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:  .LBB2_5:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -32(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    and 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB2_5
+; POWERPC64-NEXT:  # %bb.6:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_and64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    li 4, 0
+; POWERPC-NEXT:    li 6, 3
+; POWERPC-NEXT:    li 5, 2
+; POWERPC-NEXT:    stw 3, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_and_8
+; POWERPC-NEXT:    li 6, 5
+; POWERPC-NEXT:    lwz 5, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 16(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 12(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_and_8
+; POWERPC-NEXT:    lwz 5, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 8(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 8(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_fetch_and_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw and  i64* @sc64, i64 3 acquire
+  %t2 = atomicrmw and  i64* @sc64, i64 5 acquire
+  %t3 = atomicrmw and  i64* @sc64, i64 %t2 acquire
+  ret void
+}
+
+define void @atomic_fetch_or64() nounwind {
+; POWERPC64-LABEL: atomic_fetch_or64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 3, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; POWERPC64-NEXT:    li 4, 3
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB3_1:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    or 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB3_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 5
+; POWERPC64-NEXT:    std 3, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB3_3:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    or 6, 5, 4
+; POWERPC64-NEXT:    stdcx. 6, 0, 3
+; POWERPC64-NEXT:    std 4, -32(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    bne 0, .LBB3_3
+; POWERPC64-NEXT:  # %bb.4:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:  .LBB3_5:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -32(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    or 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB3_5
+; POWERPC64-NEXT:  # %bb.6:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_or64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    li 4, 0
+; POWERPC-NEXT:    li 6, 3
+; POWERPC-NEXT:    li 5, 2
+; POWERPC-NEXT:    stw 3, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_or_8
+; POWERPC-NEXT:    li 6, 5
+; POWERPC-NEXT:    lwz 5, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 16(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 12(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_or_8
+; POWERPC-NEXT:    lwz 5, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 8(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 8(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_fetch_or_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw or   i64* @sc64, i64 3 acquire
+  %t2 = atomicrmw or   i64* @sc64, i64 5 acquire
+  %t3 = atomicrmw or   i64* @sc64, i64 %t2 acquire
+  ret void
+}
+
+define void @atomic_fetch_xor64() nounwind {
+; POWERPC64-LABEL: atomic_fetch_xor64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 3, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; POWERPC64-NEXT:    li 4, 3
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB4_1:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    xor 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB4_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    li 3, 5
+; POWERPC64-NEXT:    std 3, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB4_3:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    xor 6, 5, 4
+; POWERPC64-NEXT:    stdcx. 6, 0, 3
+; POWERPC64-NEXT:    std 4, -32(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    bne 0, .LBB4_3
+; POWERPC64-NEXT:  # %bb.4:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:  .LBB4_5:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -32(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    xor 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB4_5
+; POWERPC64-NEXT:  # %bb.6:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_xor64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    li 4, 0
+; POWERPC-NEXT:    li 6, 3
+; POWERPC-NEXT:    li 5, 2
+; POWERPC-NEXT:    stw 3, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_xor_8
+; POWERPC-NEXT:    li 6, 5
+; POWERPC-NEXT:    lwz 5, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 16(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 4, 12(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    bl __atomic_fetch_xor_8
+; POWERPC-NEXT:    lwz 5, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 3, 8(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 5
+; POWERPC-NEXT:    lwz 5, 8(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    lwz 7, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_fetch_xor_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw xor  i64* @sc64, i64 3 acquire
+  %t2 = atomicrmw xor  i64* @sc64, i64 5 acquire
+  %t3 = atomicrmw xor  i64* @sc64, i64 %t2 acquire
+  ret void
+}
+
+define void @atomic_fetch_nand64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_nand64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB5_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    nand 4, 5, 4
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB5_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_nand64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    mr 6, 3
+; POWERPC-NEXT:    lis 7, sc64@ha
+; POWERPC-NEXT:    la 7, sc64@l(7)
+; POWERPC-NEXT:    li 8, 2
+; POWERPC-NEXT:    stw 3, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 7
+; POWERPC-NEXT:    lwz 7, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 7
+; POWERPC-NEXT:    stw 6, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    mr 7, 8
+; POWERPC-NEXT:    bl __atomic_fetch_nand_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw nand i64* @sc64, i64 %x acquire
+  ret void
+}
+
+define void @atomic_fetch_max64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_max64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB6_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    cmpd 5, 4
+; POWERPC64-NEXT:    ble 0, .LBB6_3
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ld 4, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 3, 0, 4
+; POWERPC64-NEXT:    bne 0, .LBB6_1
+; POWERPC64-NEXT:  .LBB6_3:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_max64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -48(1)
+; POWERPC-NEXT:    lis 5, sc64@ha
+; POWERPC-NEXT:    lwzu 6, sc64@l(5)
+; POWERPC-NEXT:    lwz 5, 4(5)
+; POWERPC-NEXT:    stw 4, 36(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 3, 32(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 6, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    b .LBB6_1
+; POWERPC-NEXT:  .LBB6_1: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 3, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 4, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 5, 32(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmpw 4, 5
+; POWERPC-NEXT:    li 6, 0
+; POWERPC-NEXT:    li 7, 1
+; POWERPC-NEXT:    bc 12, 1, .LBB6_3
+; POWERPC-NEXT:  # %bb.2: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 8, 6, 0
+; POWERPC-NEXT:    b .LBB6_4
+; POWERPC-NEXT:  .LBB6_3: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 8, 7, 0
+; POWERPC-NEXT:  .LBB6_4: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 9, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmplw 3, 9
+; POWERPC-NEXT:    bc 12, 1, .LBB6_5
+; POWERPC-NEXT:    b .LBB6_6
+; POWERPC-NEXT:  .LBB6_5: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 6, 7, 0
+; POWERPC-NEXT:  .LBB6_6: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    cmplw 4, 5
+; POWERPC-NEXT:    bc 12, 2, .LBB6_8
+; POWERPC-NEXT:  # %bb.7: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 6, 8, 0
+; POWERPC-NEXT:    b .LBB6_8
+; POWERPC-NEXT:  .LBB6_8: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    cmplwi 6, 0
+; POWERPC-NEXT:    bc 12, 2, .LBB6_10
+; POWERPC-NEXT:  # %bb.9: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 5, 4, 0
+; POWERPC-NEXT:    ori 6, 3, 0
+; POWERPC-NEXT:    b .LBB6_11
+; POWERPC-NEXT:  .LBB6_10: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 6, 9, 0
+; POWERPC-NEXT:  .LBB6_11: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    stw 4, 40(1)
+; POWERPC-NEXT:    stw 3, 44(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    addi 4, 1, 40
+; POWERPC-NEXT:    li 7, 2
+; POWERPC-NEXT:    stw 7, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    lwz 8, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_compare_exchange_8
+; POWERPC-NEXT:    lwz 4, 44(1)
+; POWERPC-NEXT:    lwz 5, 40(1)
+; POWERPC-NEXT:    cmplwi 3, 0
+; POWERPC-NEXT:    stw 5, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 4, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    beq 0, .LBB6_1
+; POWERPC-NEXT:    b .LBB6_12
+; POWERPC-NEXT:  .LBB6_12: # %atomicrmw.end
+; POWERPC-NEXT:    lwz 0, 52(1)
+; POWERPC-NEXT:    addi 1, 1, 48
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw max  i64* @sc64, i64 %x acquire
+
+  ret void
+}
+
+define void @atomic_fetch_min64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_min64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB7_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    cmpd 5, 4
+; POWERPC64-NEXT:    bge 0, .LBB7_3
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ld 4, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 3, 0, 4
+; POWERPC64-NEXT:    bne 0, .LBB7_1
+; POWERPC64-NEXT:  .LBB7_3:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_min64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -48(1)
+; POWERPC-NEXT:    lis 5, sc64@ha
+; POWERPC-NEXT:    lwzu 6, sc64@l(5)
+; POWERPC-NEXT:    lwz 5, 4(5)
+; POWERPC-NEXT:    stw 4, 36(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 3, 32(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 6, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    b .LBB7_1
+; POWERPC-NEXT:  .LBB7_1: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 3, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 4, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 5, 32(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmpw 4, 5
+; POWERPC-NEXT:    li 6, 0
+; POWERPC-NEXT:    li 7, 1
+; POWERPC-NEXT:    bc 12, 1, .LBB7_3
+; POWERPC-NEXT:  # %bb.2: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 8, 7, 0
+; POWERPC-NEXT:    b .LBB7_4
+; POWERPC-NEXT:  .LBB7_3: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 8, 6, 0
+; POWERPC-NEXT:  .LBB7_4: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 9, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmplw 3, 9
+; POWERPC-NEXT:    bc 12, 1, .LBB7_6
+; POWERPC-NEXT:  # %bb.5: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 6, 7, 0
+; POWERPC-NEXT:    b .LBB7_6
+; POWERPC-NEXT:  .LBB7_6: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    cmplw 4, 5
+; POWERPC-NEXT:    bc 12, 2, .LBB7_8
+; POWERPC-NEXT:  # %bb.7: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 6, 8, 0
+; POWERPC-NEXT:    b .LBB7_8
+; POWERPC-NEXT:  .LBB7_8: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    cmplwi 6, 0
+; POWERPC-NEXT:    bc 12, 2, .LBB7_10
+; POWERPC-NEXT:  # %bb.9: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 5, 4, 0
+; POWERPC-NEXT:    ori 6, 3, 0
+; POWERPC-NEXT:    b .LBB7_11
+; POWERPC-NEXT:  .LBB7_10: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 6, 9, 0
+; POWERPC-NEXT:  .LBB7_11: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    stw 4, 40(1)
+; POWERPC-NEXT:    stw 3, 44(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    addi 4, 1, 40
+; POWERPC-NEXT:    li 7, 2
+; POWERPC-NEXT:    stw 7, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    lwz 8, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_compare_exchange_8
+; POWERPC-NEXT:    lwz 4, 44(1)
+; POWERPC-NEXT:    lwz 5, 40(1)
+; POWERPC-NEXT:    cmplwi 3, 0
+; POWERPC-NEXT:    stw 5, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 4, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    beq 0, .LBB7_1
+; POWERPC-NEXT:    b .LBB7_12
+; POWERPC-NEXT:  .LBB7_12: # %atomicrmw.end
+; POWERPC-NEXT:    lwz 0, 52(1)
+; POWERPC-NEXT:    addi 1, 1, 48
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw min  i64* @sc64, i64 %x acquire
+
+  ret void
+}
+
+define void @atomic_fetch_umax64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_umax64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB8_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    cmpld 5, 4
+; POWERPC64-NEXT:    ble 0, .LBB8_3
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ld 4, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 3, 0, 4
+; POWERPC64-NEXT:    bne 0, .LBB8_1
+; POWERPC64-NEXT:  .LBB8_3:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_umax64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -48(1)
+; POWERPC-NEXT:    lis 5, sc64@ha
+; POWERPC-NEXT:    lwzu 6, sc64@l(5)
+; POWERPC-NEXT:    lwz 5, 4(5)
+; POWERPC-NEXT:    stw 4, 36(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 3, 32(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 6, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    b .LBB8_1
+; POWERPC-NEXT:  .LBB8_1: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 3, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 4, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 5, 32(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmplw 4, 5
+; POWERPC-NEXT:    li 6, 0
+; POWERPC-NEXT:    li 7, 1
+; POWERPC-NEXT:    bc 12, 1, .LBB8_3
+; POWERPC-NEXT:  # %bb.2: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 8, 6, 0
+; POWERPC-NEXT:    b .LBB8_4
+; POWERPC-NEXT:  .LBB8_3: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 8, 7, 0
+; POWERPC-NEXT:  .LBB8_4: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 9, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmplw 1, 3, 9
+; POWERPC-NEXT:    bc 12, 5, .LBB8_5
+; POWERPC-NEXT:    b .LBB8_6
+; POWERPC-NEXT:  .LBB8_5: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 6, 7, 0
+; POWERPC-NEXT:  .LBB8_6: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    bc 12, 2, .LBB8_8
+; POWERPC-NEXT:  # %bb.7: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 6, 8, 0
+; POWERPC-NEXT:    b .LBB8_8
+; POWERPC-NEXT:  .LBB8_8: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    cmplwi 6, 0
+; POWERPC-NEXT:    bc 12, 2, .LBB8_10
+; POWERPC-NEXT:  # %bb.9: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 5, 4, 0
+; POWERPC-NEXT:    ori 6, 3, 0
+; POWERPC-NEXT:    b .LBB8_11
+; POWERPC-NEXT:  .LBB8_10: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 6, 9, 0
+; POWERPC-NEXT:  .LBB8_11: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    stw 4, 40(1)
+; POWERPC-NEXT:    stw 3, 44(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    addi 4, 1, 40
+; POWERPC-NEXT:    li 7, 2
+; POWERPC-NEXT:    stw 7, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    lwz 8, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_compare_exchange_8
+; POWERPC-NEXT:    lwz 4, 44(1)
+; POWERPC-NEXT:    lwz 5, 40(1)
+; POWERPC-NEXT:    cmplwi 3, 0
+; POWERPC-NEXT:    stw 5, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 4, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    beq 0, .LBB8_1
+; POWERPC-NEXT:    b .LBB8_12
+; POWERPC-NEXT:  .LBB8_12: # %atomicrmw.end
+; POWERPC-NEXT:    lwz 0, 52(1)
+; POWERPC-NEXT:    addi 1, 1, 48
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw umax i64* @sc64, i64 %x acquire
+
+  ret void
+}
+
+define void @atomic_fetch_umin64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_umin64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB9_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    cmpld 5, 4
+; POWERPC64-NEXT:    bge 0, .LBB9_3
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ld 4, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 3, 0, 4
+; POWERPC64-NEXT:    bne 0, .LBB9_1
+; POWERPC64-NEXT:  .LBB9_3:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_umin64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -48(1)
+; POWERPC-NEXT:    lis 5, sc64@ha
+; POWERPC-NEXT:    lwzu 6, sc64@l(5)
+; POWERPC-NEXT:    lwz 5, 4(5)
+; POWERPC-NEXT:    stw 4, 36(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 3, 32(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 6, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    b .LBB9_1
+; POWERPC-NEXT:  .LBB9_1: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 3, 24(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 4, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    lwz 5, 32(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmplw 4, 5
+; POWERPC-NEXT:    li 6, 0
+; POWERPC-NEXT:    li 7, 1
+; POWERPC-NEXT:    bc 12, 1, .LBB9_3
+; POWERPC-NEXT:  # %bb.2: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 8, 7, 0
+; POWERPC-NEXT:    b .LBB9_4
+; POWERPC-NEXT:  .LBB9_3: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 8, 6, 0
+; POWERPC-NEXT:  .LBB9_4: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    lwz 9, 36(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    cmplw 1, 3, 9
+; POWERPC-NEXT:    bc 12, 5, .LBB9_6
+; POWERPC-NEXT:  # %bb.5: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 6, 7, 0
+; POWERPC-NEXT:    b .LBB9_6
+; POWERPC-NEXT:  .LBB9_6: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    bc 12, 2, .LBB9_8
+; POWERPC-NEXT:  # %bb.7: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 6, 8, 0
+; POWERPC-NEXT:    b .LBB9_8
+; POWERPC-NEXT:  .LBB9_8: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    cmplwi 6, 0
+; POWERPC-NEXT:    bc 12, 2, .LBB9_10
+; POWERPC-NEXT:  # %bb.9: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    ori 5, 4, 0
+; POWERPC-NEXT:    ori 6, 3, 0
+; POWERPC-NEXT:    b .LBB9_11
+; POWERPC-NEXT:  .LBB9_10: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    addi 6, 9, 0
+; POWERPC-NEXT:  .LBB9_11: # %atomicrmw.start
+; POWERPC-NEXT:    #
+; POWERPC-NEXT:    stw 4, 40(1)
+; POWERPC-NEXT:    stw 3, 44(1)
+; POWERPC-NEXT:    lis 3, sc64@ha
+; POWERPC-NEXT:    la 3, sc64@l(3)
+; POWERPC-NEXT:    addi 4, 1, 40
+; POWERPC-NEXT:    li 7, 2
+; POWERPC-NEXT:    stw 7, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    lwz 8, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_compare_exchange_8
+; POWERPC-NEXT:    lwz 4, 44(1)
+; POWERPC-NEXT:    lwz 5, 40(1)
+; POWERPC-NEXT:    cmplwi 3, 0
+; POWERPC-NEXT:    stw 5, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    stw 4, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    beq 0, .LBB9_1
+; POWERPC-NEXT:    b .LBB9_12
+; POWERPC-NEXT:  .LBB9_12: # %atomicrmw.end
+; POWERPC-NEXT:    lwz 0, 52(1)
+; POWERPC-NEXT:    addi 1, 1, 48
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw umin i64* @sc64, i64 %x acquire
+
+  ret void
+}
+
+define void @atomic_fetch_cmpxchg64() nounwind {
+; POWERPC64-LABEL: atomic_fetch_cmpxchg64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 3, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC0@toc@l(3)
+; POWERPC64-NEXT:    li 4, 1
+; POWERPC64-NEXT:    li 5, 0
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 5, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB10_1:
+; POWERPC64-NEXT:    ld 3, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 5, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    cmpd 5, 4
+; POWERPC64-NEXT:    std 4, -32(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    bne 0, .LBB10_3
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ld 4, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 3, 0, 4
+; POWERPC64-NEXT:    bne 0, .LBB10_1
+; POWERPC64-NEXT:    b .LBB10_4
+; POWERPC64-NEXT:  .LBB10_3:
+; POWERPC64-NEXT:    ld 3, -32(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ld 4, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 3, 0, 4
+; POWERPC64-NEXT:  .LBB10_4:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_cmpxchg64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    li 3, 0
+; POWERPC-NEXT:    stw 3, 28(1)
+; POWERPC-NEXT:    stw 3, 24(1)
+; POWERPC-NEXT:    lis 4, sc64@ha
+; POWERPC-NEXT:    la 4, sc64@l(4)
+; POWERPC-NEXT:    addi 5, 1, 24
+; POWERPC-NEXT:    li 6, 1
+; POWERPC-NEXT:    li 7, 2
+; POWERPC-NEXT:    stw 3, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 4
+; POWERPC-NEXT:    mr 4, 5
+; POWERPC-NEXT:    lwz 5, 20(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 7, 16(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    lwz 8, 16(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    bl __atomic_compare_exchange_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = cmpxchg i64* @sc64, i64 0, i64 1 acquire acquire
+  ret void
+}
+
+define void @atomic_fetch_store64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_store64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, 0(4)
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_store64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    mr 6, 3
+; POWERPC-NEXT:    lis 7, sc64@ha
+; POWERPC-NEXT:    la 7, sc64@l(7)
+; POWERPC-NEXT:    li 8, 3
+; POWERPC-NEXT:    stw 3, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 7
+; POWERPC-NEXT:    lwz 7, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 7
+; POWERPC-NEXT:    stw 6, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    mr 7, 8
+; POWERPC-NEXT:    bl __atomic_store_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  store atomic i64 %x, i64* @sc64 release, align 8
+  ret void
+}
+
+define void @atomic_fetch_swap64(i64 %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_swap64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    addis 4, 2, .LC0@toc@ha
+; POWERPC64-NEXT:    ld 4, .LC0@toc@l(4)
+; POWERPC64-NEXT:    std 3, -8(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB12_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 4, -8(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB12_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_swap64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -32(1)
+; POWERPC-NEXT:    mr 5, 4
+; POWERPC-NEXT:    mr 6, 3
+; POWERPC-NEXT:    lis 7, sc64@ha
+; POWERPC-NEXT:    la 7, sc64@l(7)
+; POWERPC-NEXT:    li 8, 2
+; POWERPC-NEXT:    stw 3, 28(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 3, 7
+; POWERPC-NEXT:    lwz 7, 28(1) # 4-byte Folded Reload
+; POWERPC-NEXT:    stw 5, 24(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 5, 7
+; POWERPC-NEXT:    stw 6, 20(1) # 4-byte Folded Spill
+; POWERPC-NEXT:    mr 6, 4
+; POWERPC-NEXT:    mr 7, 8
+; POWERPC-NEXT:    bl __atomic_exchange_8
+; POWERPC-NEXT:    lwz 0, 36(1)
+; POWERPC-NEXT:    addi 1, 1, 32
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw xchg i64* @sc64, i64 %x acquire
+  ret void
+}
+
+define void @atomic_fetch_swapf64(double %x) nounwind {
+; POWERPC64-LABEL: atomic_fetch_swapf64:
+; POWERPC64:       # %bb.0:
+; POWERPC64-NEXT:    stfd 1, -8(1)
+; POWERPC64-NEXT:    addis 3, 2, .LC1@toc@ha
+; POWERPC64-NEXT:    ld 3, .LC1@toc@l(3)
+; POWERPC64-NEXT:    ld 4, -8(1)
+; POWERPC64-NEXT:    std 3, -16(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:    std 4, -24(1) # 8-byte Folded Spill
+; POWERPC64-NEXT:  .LBB13_1:
+; POWERPC64-NEXT:    ld 3, -16(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    ldarx 4, 0, 3
+; POWERPC64-NEXT:    ld 4, -24(1) # 8-byte Folded Reload
+; POWERPC64-NEXT:    stdcx. 4, 0, 3
+; POWERPC64-NEXT:    bne 0, .LBB13_1
+; POWERPC64-NEXT:  # %bb.2:
+; POWERPC64-NEXT:    lwsync
+; POWERPC64-NEXT:    blr
+;
+; POWERPC-LABEL: atomic_fetch_swapf64:
+; POWERPC:       # %bb.0:
+; POWERPC-NEXT:    mflr 0
+; POWERPC-NEXT:    stw 0, 4(1)
+; POWERPC-NEXT:    stwu 1, -16(1)
+; POWERPC-NEXT:    stfd 1, 8(1)
+; POWERPC-NEXT:    lwz 5, 8(1)
+; POWERPC-NEXT:    lwz 6, 12(1)
+; POWERPC-NEXT:    lis 3, fsc64@ha
+; POWERPC-NEXT:    la 3, fsc64@l(3)
+; POWERPC-NEXT:    li 7, 2
+; POWERPC-NEXT:    bl __atomic_exchange_8
+; POWERPC-NEXT:    lwz 0, 20(1)
+; POWERPC-NEXT:    addi 1, 1, 16
+; POWERPC-NEXT:    mtlr 0
+; POWERPC-NEXT:    blr
+  %t1 = atomicrmw xchg double* @fsc64, double %x acquire
+  ret void
+}