Allow code motion (and thus folding) for atomic (but unordered) memory operands

[reames]

Summary

Building on the work done in D57601, now that we can distinguish between atomic and volatile memory accesses, go ahead and allow code motion of unordered atomics. As seen in the diffs, this allows much better folding of memory operations into using instructions. (Mostly done by the PeepholeOpt pass.)

Note: I have not reviewed all callers of hasOrderedMemoryRef since one of them - isSafeToMove - is very widely used. I'm relying on the documented semantics of each method to judge correctness.

[jfb]

Overall this looks good.
Do you have tests with intervening fences (both atomic and thread), to make sure they block motion as expected?

I'm trying to figure out if this optimization is only correct for memory_order_java, or if it also applies to memory_order_relaxed. I can't see a case in your examples which wouldn't apply to memory_order_relaxed as well. @__simt__ WDYT?
I'm not asking you to do that work. It just seems like a valid follow-up.

[reames]

In D59345#1429273, @jfb wrote:

Overall this looks good.
Do you have tests with intervening fences (both atomic and thread), to make sure they block motion as expected?

I don't, but will add some before submitting. Good idea.

I'm trying to figure out if this optimization is only correct for memory_order_java, or if it also applies to memory_order_relaxed. I can't see a case in your examples which wouldn't apply to memory_order_relaxed as well. @__simt__ WDYT?
I'm not asking you to do that work. It just seems like a valid follow-up.

I'm not sure. I believe that memory_order_relaxed maps to llvm's monotonic right? If so, then I *think* moving one is safe, but monotonic is just subtly different enough from unordered and not atomic that a more careful audit would be needed.

[jfb]

I'm trying to figure out if this optimization is only correct for memory_order_java, or if it also applies to memory_order_relaxed. I can't see a case in your examples which wouldn't apply to memory_order_relaxed as well. @__simt__ WDYT?
I'm not asking you to do that work. It just seems like a valid follow-up.

I'm not sure. I believe that memory_order_relaxed maps to llvm's monotonic right? If so, then I *think* moving one is safe, but monotonic is just subtly different enough from unordered and not atomic that a more careful audit would be needed.

Yes, memory_order_relaxed is monotonic.

llvm/trunk/lib/CodeGen/MachineInstr.cpp

if (!mayStore() &&
     return false;
 
   // Otherwise, if the instruction has no memory reference information,
   // conservatively assume it wasn't preserved.
   if (memoperands_empty())
     return true;
 
   // Check if any of our memory operands are ordered.
-  // TODO: This should probably be be isUnordered (see D57601), but the callers
-  // need audited and test cases written to be sure.
   return llvm::any_of(memoperands(), [](const MachineMemOperand *MMO) {
-    return MMO->isVolatile() || MMO->isAtomic();
+    return !MMO->isUnordered();
   });
 }
 
 /// isDereferenceableInvariantLoad - Return true if this instruction will never
 /// trap and is loading from a location whose value is invariant across a run of
 /// this function.
 bool MachineInstr::isDereferenceableInvariantLoad(AliasAnalysis *AA) const {
   // If the instruction doesn't load at all, it isn't an invariant load.

llvm/trunk/test/CodeGen/X86/atomic-non-integer.ll

; CHECK-NEXT:    retq
   ret void
 }
 
 define half @load_half(half* %fptr) {
 ; CHECK-LABEL: load_half:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    movzwl (%rdi), %eax
-; CHECK-NEXT:    movzwl %ax, %edi
+; CHECK-NEXT:    movzwl (%rdi), %edi
 ; CHECK-NEXT:    callq __gnu_h2f_ieee
 ; CHECK-NEXT:    popq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
   %v = load atomic half, half* %fptr unordered, align 2
   ret half %v
 }
 
 define float @load_float(float* %fptr) {
 ; CHECK-LABEL: load_float:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl (%rdi), %eax
-; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    movd (%rdi), %xmm0
 ; CHECK-NEXT:    retq
   %v = load atomic float, float* %fptr unordered, align 4
   ret float %v
 }
 
 define double @load_double(double* %fptr) {
 ; CHECK-LABEL: load_double:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    movq %rax, %xmm0
+; CHECK-NEXT:    movq (%rdi), %xmm0
 ; CHECK-NEXT:    retq
   %v = load atomic double, double* %fptr unordered, align 8
   ret double %v
 }
 
 define fp128 @load_fp128(fp128* %fptr) {
 ; CHECK-LABEL: load_fp128:
 ; CHECK:       # %bb.0:

llvm/trunk/test/CodeGen/X86/atomic-unordered.ll

 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    addq $15, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = add i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, as expected
 define i64 @load_fold_add2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_add2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addq (%rdi), %rsi
 ; CHECK-O0-NEXT:    movq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_add2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    addq %rsi, %rax
+; CHECK-O3-NEXT:    movq %rsi, %rax
+; CHECK-O3-NEXT:    addq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = add i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_add3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_add3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    addq (%rsi), %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_add3:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rcx
 ; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    addq %rcx, %rax
+; CHECK-O3-NEXT:    addq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = add i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal, as expected

 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    addq $-15, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = sub i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, as expected
 define i64 @load_fold_sub2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_sub2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    subq %rsi, %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_sub2:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    subq %rsi, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = sub i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_sub3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_sub3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    subq (%rsi), %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_sub3:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq (%rsi), %rcx
-; CHECK-O3-NEXT:    subq %rcx, %rax
+; CHECK-O3-NEXT:    subq (%rsi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = sub i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal, as expected

 ; CHECK-O3-NEXT:    leaq (%rax,%rax,4), %rax
 ; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = mul i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, O0 is better than O3 codegen (TODO)
 define i64 @load_fold_mul2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_mul2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    imulq (%rdi), %rsi
 ; CHECK-O0-NEXT:    movq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_mul2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    imulq %rsi, %rax
+; CHECK-O3-NEXT:    movq %rsi, %rax
+; CHECK-O3-NEXT:    imulq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = mul i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_mul3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_mul3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    imulq (%rsi), %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_mul3:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rcx
 ; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    imulq %rcx, %rax
+; CHECK-O3-NEXT:    imulq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = mul i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal to fold (TODO)

 ; CHECK-O3-NEXT:    cqto
 ; CHECK-O3-NEXT:    idivq %rsi
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = sdiv i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_sdiv3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_sdiv3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    cqto
 ; CHECK-O0-NEXT:    idivq (%rsi)
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_sdiv3:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq (%rsi), %rcx
 ; CHECK-O3-NEXT:    cqto
-; CHECK-O3-NEXT:    idivq %rcx
+; CHECK-O3-NEXT:    idivq (%rsi)
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = sdiv i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal to fold (TODO)

 ; CHECK-O3-NEXT:    movq %rdx, %rax
 ; CHECK-O3-NEXT:    shrq $3, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = udiv i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, as expected
 define i64 @load_fold_udiv2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_udiv2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    movl %ecx, %edx
 ; CHECK-O0-NEXT:    divq %rsi
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_udiv2:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    xorl %edx, %edx
 ; CHECK-O3-NEXT:    divq %rsi
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = udiv i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_udiv3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_udiv3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    movl %ecx, %edx
 ; CHECK-O0-NEXT:    divq (%rsi)
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_udiv3:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq (%rsi), %rcx
 ; CHECK-O3-NEXT:    xorl %edx, %edx
-; CHECK-O3-NEXT:    divq %rcx
+; CHECK-O3-NEXT:    divq (%rsi)
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = udiv i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal to fold (TODO)

 ; CHECK-O3-NEXT:    idivq %rsi
 ; CHECK-O3-NEXT:    movq %rdx, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = srem i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_srem3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_srem3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    cqto
 ; CHECK-O0-NEXT:    idivq (%rsi)
 ; CHECK-O0-NEXT:    movq %rdx, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_srem3:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq (%rsi), %rcx
 ; CHECK-O3-NEXT:    cqto
-; CHECK-O3-NEXT:    idivq %rcx
+; CHECK-O3-NEXT:    idivq (%rsi)
 ; CHECK-O3-NEXT:    movq %rdx, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = srem i64 %v, %v2
   ret i64 %ret
 }
 

 ; CHECK-O3-NEXT:    divq %rsi
 ; CHECK-O3-NEXT:    movq %rdx, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = urem i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_urem3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_urem3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    movl %ecx, %edx
 ; CHECK-O0-NEXT:    divq (%rsi)
 ; CHECK-O0-NEXT:    movq %rdx, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_urem3:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq (%rsi), %rcx
 ; CHECK-O3-NEXT:    xorl %edx, %edx
-; CHECK-O3-NEXT:    divq %rcx
+; CHECK-O3-NEXT:    divq (%rsi)
 ; CHECK-O3-NEXT:    movq %rdx, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = urem i64 %v, %v2
   ret i64 %ret
 }
 

 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    andl $15, %eax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = and i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, as expected
 define i64 @load_fold_and2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_and2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    andq (%rdi), %rsi
 ; CHECK-O0-NEXT:    movq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_and2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    andq %rsi, %rax
+; CHECK-O3-NEXT:    movq %rsi, %rax
+; CHECK-O3-NEXT:    andq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = and i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_and3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_and3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    andq (%rsi), %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_and3:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rcx
 ; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    andq %rcx, %rax
+; CHECK-O3-NEXT:    andq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = and i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal, as expected

 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    orq $15, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = or i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, as expected
 define i64 @load_fold_or2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_or2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    orq (%rdi), %rsi
 ; CHECK-O0-NEXT:    movq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_or2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    orq %rsi, %rax
+; CHECK-O3-NEXT:    movq %rsi, %rax
+; CHECK-O3-NEXT:    orq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = or i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_or3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_or3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    orq (%rsi), %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_or3:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rcx
 ; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    orq %rcx, %rax
+; CHECK-O3-NEXT:    orq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = or i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal, as expected

 ; CHECK-O3-NEXT:    movq (%rdi), %rax
 ; CHECK-O3-NEXT:    xorq $15, %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = xor i64 %v, 15
   ret i64 %ret
 }
 
-; Legal, as expected
 define i64 @load_fold_xor2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_xor2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    xorq (%rdi), %rsi
 ; CHECK-O0-NEXT:    movq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_xor2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    xorq %rsi, %rax
+; CHECK-O3-NEXT:    movq %rsi, %rax
+; CHECK-O3-NEXT:    xorq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = xor i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i64 @load_fold_xor3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_xor3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    xorq (%rsi), %rdi
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_xor3:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rcx
 ; CHECK-O3-NEXT:    movq (%rsi), %rax
-; CHECK-O3-NEXT:    xorq %rcx, %rax
+; CHECK-O3-NEXT:    xorq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = xor i64 %v, %v2
   ret i64 %ret
 }
 
-; Legal to fold (TODO)
 define i1 @load_fold_icmp1(i64* %p) {
 ; CHECK-O0-LABEL: load_fold_icmp1:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    subq $15, %rdi
 ; CHECK-O0-NEXT:    sete %al
 ; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_icmp1:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    cmpq $15, %rax
+; CHECK-O3-NEXT:    cmpq $15, (%rdi)
 ; CHECK-O3-NEXT:    sete %al
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = icmp eq i64 %v, 15
   ret i1 %ret
 }
 
-; Legal to fold (TODO)
 define i1 @load_fold_icmp2(i64* %p, i64 %v2) {
 ; CHECK-O0-LABEL: load_fold_icmp2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    subq %rsi, %rdi
 ; CHECK-O0-NEXT:    sete %al
 ; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_icmp2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    cmpq %rsi, %rax
+; CHECK-O3-NEXT:    cmpq %rsi, (%rdi)
 ; CHECK-O3-NEXT:    sete %al
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %ret = icmp eq i64 %v, %v2
   ret i1 %ret
 }
 
-; Legal to fold (TODO)
 define i1 @load_fold_icmp3(i64* %p1, i64* %p2) {
 ; CHECK-O0-LABEL: load_fold_icmp3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rdi
 ; CHECK-O0-NEXT:    movq (%rsi), %rsi
 ; CHECK-O0-NEXT:    subq %rsi, %rdi
 ; CHECK-O0-NEXT:    sete %al
 ; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_fold_icmp3:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq (%rsi), %rcx
-; CHECK-O3-NEXT:    cmpq %rcx, %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rax
+; CHECK-O3-NEXT:    cmpq %rax, (%rdi)
 ; CHECK-O3-NEXT:    sete %al
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p1 unordered, align 8
   %v2 = load atomic i64, i64* %p2 unordered, align 8
   %ret = icmp eq i64 %v, %v2
   ret i1 %ret
 }
 

 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    imulq %rsi, %rax
 ; CHECK-O0-NEXT:    movq %rax, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: rmw_fold_mul2:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    imulq %rsi, %rax
-; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    imulq (%rdi), %rsi
+; CHECK-O3-NEXT:    movq %rsi, (%rdi)
 ; CHECK-O3-NEXT:    retq
   %prev = load atomic i64, i64* %p unordered, align 8
   %val = mul i64 %prev, %v
   store atomic i64 %val, i64* %p unordered, align 8
   ret void
 }
 
 ; Legal, as expected

 ; CHECK-O0-LABEL: load_forwarding:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
 ; CHECK-O0-NEXT:    orq (%rdi), %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: load_forwarding:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    movq (%rdi), %rcx
 ; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    orq %rcx, %rax
+; CHECK-O3-NEXT:    orq (%rdi), %rax
 ; CHECK-O3-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   %v2 = load atomic i64, i64* %p unordered, align 8
   %ret = or i64 %v, %v2
   ret i64 %ret
 }
 
 ; Legal to forward (TODO)