[X86] Make `shouldExpandLogicAtomicRMWInIR` able to match both operands.

[goldstein.w.n]

Summary

Previous logic was buggy and erroneously asserted that I->operand(0) must
be the RMW instruction. This change fixes that and makes it so that the
RMW instruction can be used in operand 0 or 1.

Also update the tests to explicitly test RMW as operand 0/1 (no change
to codegen).

[goldstein.w.n]

Make this fix complete

[goldstein.w.n]

Fix commit message

llvm/test/CodeGen/X86/atomic-rm-bit-test.ll

159

This seems to change RA (or maybe something else). Whatever it is there is some logic that is a bit faulty and make assumption about operands.

[goldstein.w.n]

Also note, the new tests are able to reproduce the assertion bug.

[pengfei]

LGTM with some nits.

llvm/lib/Target/X86/X86ISelLowering.cpp

31534–31536

Do you have a test for it? I suspect it's dead code. Maybe just use assert?

31539–31540

This can trun into auto *C1 = cast<ConstantInt>(AI->getValOperand());.

31608–31609

ditto.

[goldstein.w.n]

dyn_cast -> cast. Add test for dead and

[goldstein.w.n]

Correct align in test

llvm/lib/Target/X86/X86ISelLowering.cpp

31534–31536

Do you have a test for it? I suspect it's dead code. Maybe just use assert?

I think assert is a mistake b.c its definitely representable in IR:

define zeroext i16 @atomic_shl1_small_mask_xor_16_gpr_val(ptr %v, i16 zeroext %c) nounwind {
entry:
  %0 = and i16 %c, 7
  %shl = shl nuw nsw i16 1, %0
  %1 = atomicrmw xor ptr %v, i16 %shl monotonic, align 2
  %and = and i16 %1, %1
  ret i16 %and
}

I think its okay to miss optimizations when the IR is funky
but we shouldn't crash.

Added a test just to make sure we don't crash.

31539–31540

This can trun into auto *C1 = cast<ConstantInt>(AI->getValOperand());.

[RKSimon]

llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll

1474

I'd expect DAG canonicalization to commute this in DAGCombiner::visitAND?

[goldstein.w.n]

llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll

1474

I'd expect DAG canonicalization to commute this in DAGCombiner::visitAND?

The version that had assertions for canonical constant/non-constant operands did hit them so I expect there is some sequence where we go to shouldExpandLogicAtomicRMWInIR before canonicalization.

llvm/lib/Target/X86/X86ISelLowering.cpp

X86TargetLowering::shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const {
   Instruction *I = AI->user_back();
   auto BitChange = FindSingleBitChange(AI->getValOperand());
   if (BitChange.second == UndefBit || !AI->hasOneUse() ||
       I->getOpcode() != Instruction::And ||
       AI->getType()->getPrimitiveSizeInBits() == 8 ||
       AI->getParent() != I->getParent())
     return AtomicExpansionKind::CmpXChg;
 
-  assert(I->getOperand(0) == AI);
+  unsigned OtherIdx = I->getOperand(0) == AI ? 1 : 0;
+
+  // This is a redundant AND, it should get cleaned up elsewhere.
+  if (AI == I->getOperand(OtherIdx))
+    return AtomicExpansionKind::CmpXChg;
+
   // The following instruction must be a AND single bit.
   if (BitChange.second == ConstantBit || BitChange.second == NotConstantBit) {
-    auto *C1 = dyn_cast<ConstantInt>(AI->getValOperand());
-    assert(C1 != nullptr);
-    auto *C2 = dyn_cast<ConstantInt>(I->getOperand(1));
+    auto *C1 = cast<ConstantInt>(AI->getValOperand());
+    auto *C2 = dyn_cast<ConstantInt>(I->getOperand(OtherIdx));
     if (!C2 || !isPowerOf2_64(C2->getZExtValue())) {
       return AtomicExpansionKind::CmpXChg;
     }
     if (AI->getOperation() == AtomicRMWInst::And) {
       return ~C1->getValue() == C2->getValue()
                  ? AtomicExpansionKind::BitTestIntrinsic
                  : AtomicExpansionKind::CmpXChg;
     }
     return C1 == C2 ? AtomicExpansionKind::BitTestIntrinsic
                     : AtomicExpansionKind::CmpXChg;
   }
 
   assert(BitChange.second == ShiftBit || BitChange.second == NotShiftBit);
 
-  auto BitTested = FindSingleBitChange(I->getOperand(1));
+  auto BitTested = FindSingleBitChange(I->getOperand(OtherIdx));
   if (BitTested.second != ShiftBit && BitTested.second != NotShiftBit)
     return AtomicExpansionKind::CmpXChg;
 
   assert(BitChange.first != nullptr && BitTested.first != nullptr);
 
   // If shift amounts are not the same we can't use BitTestIntrinsic.
   if (BitChange.first != BitTested.first)
     return AtomicExpansionKind::CmpXChg;
void X86TargetLowering::emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const {
   Instruction *I = AI->user_back();
   LLVMContext &Ctx = AI->getContext();
   Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
                                           Type::getInt8PtrTy(Ctx));
   Function *BitTest = nullptr;
   Value *Result = nullptr;
   auto BitTested = FindSingleBitChange(AI->getValOperand());
   assert(BitTested.first != nullptr);
+
   if (BitTested.second == ConstantBit || BitTested.second == NotConstantBit) {
-    auto *C = dyn_cast<ConstantInt>(I->getOperand(1));
-    assert(C != nullptr);
+    auto *C = cast<ConstantInt>(I->getOperand(I->getOperand(0) == AI ? 1 : 0));
 
     BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_C, AI->getType());
 
     unsigned Imm = countTrailingZeros(C->getZExtValue());
     Result = Builder.CreateCall(BitTest, {Addr, Builder.getInt8(Imm)});
   } else {
     BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_I, AI->getType());
 

llvm/test/CodeGen/X86/atomic-rm-bit-test-64.ll

 ; CHECK-NEXT:    lock btcq %rcx, (%rdi)
 ; CHECK-NEXT:    setb %al
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rax
 ; CHECK-NEXT:    retq
 entry:
   %shl = shl nuw i64 1, %c
   %0 = atomicrmw xor ptr %v, i64 %shl monotonic, align 8
-  %and = and i64 %0, %shl
+  %and = and i64 %shl, %0
   ret i64 %and
 }
 
 define i64 @atomic_shl2_xor_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl2_xor_64_gpr_val:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rsi, %rcx
 ; CHECK-NEXT:    movl $2, %edx

 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rax
 ; CHECK-NEXT:    retq
 entry:
   %rem = and i64 %c, 63
   %shl = shl nuw i64 1, %rem
   %0 = atomicrmw xor ptr %v, i64 %shl monotonic, align 8
   %shl1 = shl nuw i64 1, %c
-  %and = and i64 %0, %shl1
+  %and = and i64 %shl1, %0
   ret i64 %and
 }
 
 define i64 @atomic_shl1_mask1_xor_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl1_mask1_xor_64_gpr_val:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rsi, %rcx
 ; CHECK-NEXT:    andl $63, %ecx

 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    btq %rcx, %rax
 ; CHECK-NEXT:    setae %dl
 ; CHECK-NEXT:    movq %rdx, %rax
 ; CHECK-NEXT:    retq
 entry:
   %shl = shl nuw i64 1, %c
   %0 = atomicrmw xor ptr %v, i64 %shl monotonic, align 8
-  %1 = xor i64 %0, -1
+  %1 = sub i64 -1, %0
   %2 = lshr i64 %1, %c
   %conv = and i64 %2, 1
   ret i64 %conv
 }
 
 define i64 @atomic_shl2_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl2_xor_64_gpr_valz:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rsi, %rcx
 ; CHECK-NEXT:    movl $2, %edx
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rdx
 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB9_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rcx
 ; CHECK-NEXT:    xorq %rdx, %rcx
 ; CHECK-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; CHECK-NEXT:    jne .LBB9_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    testq %rdx, %rax
+; CHECK-NEXT:    testq %rax, %rdx
 ; CHECK-NEXT:    sete %cl
 ; CHECK-NEXT:    movq %rcx, %rax
 ; CHECK-NEXT:    retq
 entry:
   %shl = shl i64 2, %c
   %0 = atomicrmw xor ptr %v, i64 %shl monotonic, align 8
-  %and = and i64 %0, %shl
+  %and = and i64 %shl, %0
   %tobool.not = icmp eq i64 %and, 0
   %conv = zext i1 %tobool.not to i64
   ret i64 %conv
 }
 
 define i64 @atomic_shl1_neq_xor_64_gpr_valz(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl1_neq_xor_64_gpr_valz:
 ; CHECK:       # %bb.0: # %entry

 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    lock btrq %rcx, (%rdi)
 ; CHECK-NEXT:    setb %al
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rax
 ; CHECK-NEXT:    retq
 entry:
   %shl = shl nuw i64 1, %c
-  %not = xor i64 %shl, -1
+  %not = sub i64 -1, %shl
   %0 = atomicrmw and ptr %v, i64 %not monotonic, align 8
   %and = and i64 %0, %shl
   ret i64 %and
 }
 
 define i64 @atomic_shl2_and_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl2_and_64_gpr_val:
 ; CHECK:       # %bb.0: # %entry

 ; CHECK-NEXT:    lock btrq %rcx, (%rdi)
 ; CHECK-NEXT:    setb %al
 ; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    shlq %cl, %rax
 ; CHECK-NEXT:    retq
 entry:
   %rem = and i64 %c, 63
   %shl = shl nuw i64 1, %rem
-  %not = xor i64 %shl, -1
+  %not = sub i64 -1, %shl
   %0 = atomicrmw and ptr %v, i64 %not monotonic, align 8
   %shl1 = shl nuw i64 1, %c
   %and = and i64 %0, %shl1
   ret i64 %and
 }
 
 define i64 @atomic_shl1_mask1_and_64_gpr_val(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl1_mask1_and_64_gpr_val:

 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB31_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
 ; CHECK-NEXT:    andq %rdx, %rsi
 ; CHECK-NEXT:    lock cmpxchgq %rsi, (%rdi)
 ; CHECK-NEXT:    jne .LBB31_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
-; CHECK-NEXT:    andq %rcx, %rax
+; CHECK-NEXT:    andq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, %rax
 ; CHECK-NEXT:    retq
 entry:
   %sub = sub i64 0, %c
   %and = and i64 %sub, %c
   %not = xor i64 %and, -1
   %0 = atomicrmw and ptr %v, i64 %not monotonic, align 8
-  %and3 = and i64 %0, %and
+  %and3 = and i64 %and, %0
   ret i64 %and3
 }
 
 define i64 @atomic_shl1_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl1_and_64_gpr_valnz:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq %rsi, %rcx
 ; CHECK-NEXT:    movq $-2, %rdx

 ; CHECK-NEXT:  .LBB33_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    movq %rax, %rsi
 ; CHECK-NEXT:    andq %rcx, %rsi
 ; CHECK-NEXT:    lock cmpxchgq %rsi, (%rdi)
 ; CHECK-NEXT:    jne .LBB33_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    testq %rdx, %rax
+; CHECK-NEXT:    testq %rax, %rdx
 ; CHECK-NEXT:    setne %cl
 ; CHECK-NEXT:    movq %rcx, %rax
 ; CHECK-NEXT:    retq
 entry:
   %shl = shl i64 2, %c
   %not = xor i64 %shl, -1
   %0 = atomicrmw and ptr %v, i64 %not monotonic, align 8
-  %and = and i64 %0, %shl
+  %and = and i64 %shl, %0
   %tobool = icmp ne i64 %and, 0
   %conv = zext i1 %tobool to i64
   ret i64 %conv
 }
 
 define i64 @atomic_shl1_neq_and_64_gpr_valnz(ptr %v, i64 %c) nounwind {
 ; CHECK-LABEL: atomic_shl1_neq_and_64_gpr_valnz:
 ; CHECK:       # %bb.0: # %entry

 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB40_1:
 ; CHECK-NEXT:    movl $123, %eax
 ; CHECK-NEXT:    retq
 entry:
   %shl = shl nuw i64 1, %c
   %not = xor i64 %shl, -1
   %0 = atomicrmw and ptr %v, i64 %not monotonic, align 8
-  %and = and i64 %0, %shl
+  %and = and i64 %shl, %0
   %tobool.not = icmp eq i64 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %arrayidx = getelementptr inbounds i64, ptr %v, i64 %c
   %1 = load i64, ptr %arrayidx, align 8
   br label %return
 

 ; CHECK-NEXT:    movl $123, %eax
 ; CHECK-NEXT:    retq
 entry:
   %rem = and i64 %c, 63
   %shl = shl nuw i64 1, %rem
   %not = xor i64 %shl, -1
   %0 = atomicrmw and ptr %v, i64 %not monotonic, align 8
   %shl1 = shl nuw i64 1, %c
-  %and = and i64 %0, %shl1
+  %and = and i64 %shl1, %0
   %tobool.not = icmp eq i64 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %arrayidx = getelementptr inbounds i64, ptr %v, i64 %c
   %1 = load i64, ptr %arrayidx, align 8
   br label %return
 

 ; CHECK-NEXT:    je .LBB50_1
 ; CHECK-NEXT:  # %bb.2: # %return
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB50_1: # %if.then
 ; CHECK-NEXT:    movq 32(%rdi), %rax
 ; CHECK-NEXT:    retq
 entry:
   %0 = atomicrmw xor ptr %v, i64 16 monotonic, align 8
-  %and = and i64 %0, 16
+  %and = and i64 16, %0
   %tobool.not = icmp eq i64 %and, 0
   br i1 %tobool.not, label %if.then, label %return
 
 if.then:                                          ; preds = %entry
   %arrayidx = getelementptr inbounds i64, ptr %v, i64 4
   %1 = load i64, ptr %arrayidx, align 8
   br label %return
 

llvm/test/CodeGen/X86/atomic-rm-bit-test.ll

 ; X86-NEXT:    .p2align 4, 0x90
 ; X86-NEXT:  .LBB0_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    orb %dl, %cl
 ; X86-NEXT:    lock cmpxchgb %cl, (%esi)
 ; X86-NEXT:    jne .LBB0_1
 ; X86-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-NEXT:    andb %dl, %al
+; X86-NEXT:    andb %al, %dl
+; X86-NEXT:    movl %edx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: atomic_shl1_or_8_gpr_val:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl %esi, %ecx
 ; X64-NEXT:    movl $1, %edx
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %edx
 ; X64-NEXT:    movzbl (%rdi), %eax
 ; X64-NEXT:    .p2align 4, 0x90
 ; X64-NEXT:  .LBB0_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
 ; X64-NEXT:    orb %dl, %cl
 ; X64-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; X64-NEXT:    jne .LBB0_1
 ; X64-NEXT:  # %bb.2: # %atomicrmw.end
-; X64-NEXT:    andb %dl, %al
+; X64-NEXT:    andb %al, %dl
+; X64-NEXT:    movl %edx, %eax
 ; X64-NEXT:    retq
 entry:
   %conv = zext i8 %c to i32
   %shl = shl nuw i32 1, %conv
   %conv1 = trunc i32 %shl to i8
   %0 = atomicrmw or ptr %v, i8 %conv1 monotonic, align 1
-  %conv5 = and i8 %0, %conv1
+  %conv5 = and i8 %conv1, %0
   ret i8 %conv5
 }
 
 define zeroext i8 @atomic_shl1_mask0_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwind {
 ; X86-LABEL: atomic_shl1_mask0_or_8_gpr_val:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %esi
entry:
   %2 = trunc i32 %shl4 to i8
   %conv5 = and i8 %1, %2
   ret i8 %conv5
 }
 
 define zeroext i8 @atomic_shl1_mask01_or_8_gpr_val(ptr %v, i8 zeroext %c) nounwind {
 ; X86-LABEL: atomic_shl1_mask01_or_8_gpr_val:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    andb $7, %cl
-; X86-NEXT:    movb $1, %ah
-; X86-NEXT:    shlb %cl, %ah
-; X86-NEXT:    movb (%edx), %al
+; X86-NEXT:    movb $1, %dl
+; X86-NEXT:    shlb %cl, %dl
+; X86-NEXT:    movzbl (%esi), %eax
 ; X86-NEXT:    .p2align 4, 0x90
 ; X86-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    orb %ah, %cl
-; X86-NEXT:    lock cmpxchgb %cl, (%edx)
+; X86-NEXT:    orb %dl, %cl
+; X86-NEXT:    lock cmpxchgb %cl, (%esi)
 ; X86-NEXT:    jne .LBB2_1
 ; X86-NEXT:  # %bb.2: # %atomicrmw.end
-; X86-NEXT:    andb %ah, %al
+; X86-NEXT:    andb %al, %dl
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: atomic_shl1_mask01_or_8_gpr_val:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl %esi, %ecx
 ; X64-NEXT:    andb $7, %cl
 ; X64-NEXT:    movb $1, %dl
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shlb %cl, %dl
 ; X64-NEXT:    movzbl (%rdi), %eax
 ; X64-NEXT:    .p2align 4, 0x90
 ; X64-NEXT:  .LBB2_1: # %atomicrmw.start
 ; X64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X64-NEXT:    movl %eax, %ecx
 ; X64-NEXT:    orb %dl, %cl
 ; X64-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; X64-NEXT:    jne .LBB2_1
 ; X64-NEXT:  # %bb.2: # %atomicrmw.end
-; X64-NEXT:    andb %dl, %al
+; X64-NEXT:    andb %al, %dl
+; X64-NEXT:    movl %edx, %eax
 ; X64-NEXT:    retq
 entry:
   %0 = and i8 %c, 7
   %shl = shl nuw i8 1, %0
   %1 = atomicrmw or ptr %v, i8 %shl monotonic, align 1
-  %conv7 = and i8 %1, %shl
+  %conv7 = and i8 %shl, %1
   ret i8 %conv7
 }
 
 define zeroext i8 @atomic_shl1_xor_8_gpr_valz(ptr %v, i8 zeroext %c) nounwind {
 ; X86-LABEL: atomic_shl1_xor_8_gpr_valz:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi

 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 entry:
   %0 = and i16 %c, 7
   %shl = shl nuw nsw i16 1, %0
   %1 = atomicrmw xor ptr %v, i16 %shl monotonic, align 2
-  %and = and i16 %1, %shl
+  %and = and i16 %shl, %1
   ret i16 %and
 }
 
 define zeroext i16 @atomic_shl1_mask0_xor_16_gpr_val(ptr %v, i16 zeroext %c) nounwind {
 ; X86-LABEL: atomic_shl1_mask0_xor_16_gpr_val:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi

 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 entry:
   %0 = and i16 %c, 7
   %shl = shl nuw nsw i16 1, %0
   %not = xor i16 %shl, -1
   %1 = atomicrmw and ptr %v, i16 %not monotonic, align 2
-  %and = and i16 %1, %shl
+  %and = and i16 %shl, %1
   ret i16 %and
 }
 
 define zeroext i16 @atomic_shl1_mask0_and_16_gpr_val(ptr %v, i16 zeroext %c) nounwind {
 ; X86-LABEL: atomic_shl1_mask0_and_16_gpr_val:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi

 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    lock btsw $4, (%rdi)
 ; X64-NEXT:    setb %al
 ; X64-NEXT:    shll $4, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 entry:
   %0 = atomicrmw or ptr %v, i16 16 monotonic, align 2
-  %1 = and i16 %0, 16
+  %1 = and i16 16, %0
   ret i16 %1
 }
 
 define zeroext i16 @atomic_shl1_or_16_const_valnz(ptr %v) nounwind {
 ; X86-LABEL: atomic_shl1_or_16_const_valnz:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %eax

 ; X64-NEXT:    lock btsl %ecx, (%rdi)
 ; X64-NEXT:    setb %al
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    retq
 entry:
   %shl = shl nuw i32 1, %c
   %0 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %0, %shl
+  %and = and i32 %shl, %0
   ret i32 %and
 }
 
 define i32 @atomic_shl1_small_mask_or_32_gpr_val(ptr %v, i32 %c) nounwind {
 ; X86-LABEL: atomic_shl1_small_mask_or_32_gpr_val:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx

 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    retq
 entry:
   %shl = shl nuw i32 1, %c
   %0 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
   %1 = and i32 %c, 31
   %shl1 = shl nuw i32 1, %1
-  %and = and i32 %0, %shl1
+  %and = and i32 %shl1, %0
   ret i32 %and
 }
 
 define i32 @atomic_shl1_mask01_or_32_gpr_val(ptr %v, i32 %c) nounwind {
 ; X86-LABEL: atomic_shl1_mask01_or_32_gpr_val:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx

 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB79_1:
 ; X64-NEXT:    movl $123, %eax
 ; X64-NEXT:    retq
 entry:
   %0 = and i32 %c, 15
   %shl = shl nuw nsw i32 1, %0
   %1 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %1, %shl
+  %and = and i32 %shl, %1
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %conv2 = zext i32 %0 to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv2
   %2 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB82_1:
 ; X64-NEXT:    movl $123, %eax
 ; X64-NEXT:    retq
 entry:
   %rem = and i32 %c, 31
   %shl = shl nuw i32 1, %rem
   %0 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %0, %shl
+  %and = and i32 %shl, %0
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %conv = zext i32 %c to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv
   %1 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:  .LBB85_1: # %if.then
 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movl (%rdi,%rax,4), %eax
 ; X64-NEXT:    retq
 entry:
   %0 = and i32 %c, 15
   %shl = shl nuw nsw i32 1, %0
   %1 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %1, %shl
+  %and = and i32 %shl, %1
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %if.then, label %return
 
 if.then:                                          ; preds = %entry
   %conv2 = zext i32 %0 to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv2
   %2 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movl (%rdi,%rax,4), %eax
 ; X64-NEXT:    retq
 entry:
   %shl = shl nuw i32 1, %c
   %0 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
   %rem = and i32 %c, 31
   %shl1 = shl nuw i32 1, %rem
-  %and = and i32 %0, %shl1
+  %and = and i32 %shl1, %0
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %if.then, label %return
 
 if.then:                                          ; preds = %entry
   %conv = zext i32 %c to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv
   %1 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    movl (%rdi,%rax,4), %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB90_1:
 ; X64-NEXT:    movl $123, %eax
 ; X64-NEXT:    retq
 entry:
   %shl = shl nuw i32 1, %c
   %0 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %0, %shl
+  %and = and i32 %shl, %0
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %idxprom = zext i32 %c to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %idxprom
   %1 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB94_1:
 ; X64-NEXT:    movl $123, %eax
 ; X64-NEXT:    retq
 entry:
   %rem = and i32 %c, 31
   %shl = shl nuw i32 1, %rem
   %0 = atomicrmw or ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %0, %shl
+  %and = and i32 %shl, %0
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %conv = zext i32 %c to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv
   %1 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    btl %ecx, %eax
 ; X64-NEXT:    setae %dl
 ; X64-NEXT:    movl %edx, %eax
 ; X64-NEXT:    retq
 entry:
   %shl = shl nuw i32 1, %c
   %0 = atomicrmw and ptr %v, i32 %shl monotonic, align 4
   %1 = and i32 %c, 31
-  %2 = xor i32 %0, -1
+  %2 = xor i32 -1, %0
   %3 = lshr i32 %2, %1
   %lnot.ext = and i32 %3, 1
   ret i32 %lnot.ext
 }
 
 define i32 @atomic_shl1_mask01_and_32_gpr_valz(ptr %v, i32 %c) nounwind {
 ; X86-LABEL: atomic_shl1_mask01_and_32_gpr_valz:
 ; X86:       # %bb.0: # %entry

 ; X64-NEXT:  .LBB104_3:
 ; X64-NEXT:    movl $123, %eax
 ; X64-NEXT:    retq
 entry:
   %rem = and i32 %c, 31
   %shl = shl nuw i32 1, %rem
   %0 = atomicrmw and ptr %v, i32 %shl monotonic, align 4
   %shl1 = shl nuw i32 1, %c
-  %and = and i32 %0, %shl1
+  %and = and i32 %shl1, %0
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %conv = zext i32 %c to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv
   %1 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movl (%rdi,%rax,4), %eax
 ; X64-NEXT:  .LBB107_2: # %return
 ; X64-NEXT:    retq
 entry:
   %sub = sub i32 0, %c
   %and = and i32 %sub, %c
   %0 = atomicrmw and ptr %v, i32 %and monotonic, align 4
-  %and3 = and i32 %0, %and
+  %and3 = and i32 %and, %0
   %tobool.not = icmp eq i32 %and3, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %idxprom = zext i32 %c to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %idxprom
   %1 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64-NEXT:    movl %ecx, %eax
 ; X64-NEXT:    movl (%rdi,%rax,4), %eax
 ; X64-NEXT:  .LBB109_2: # %return
 ; X64-NEXT:    retq
 entry:
   %0 = and i32 %c, 15
   %shl = shl nuw nsw i32 1, %0
   %1 = atomicrmw and ptr %v, i32 %shl monotonic, align 4
-  %and = and i32 %1, %shl
+  %and = and i32 %shl, %1
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %if.then, label %return
 
 if.then:                                          ; preds = %entry
   %conv2 = zext i32 %0 to i64
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 %conv2
   %2 = load i32, ptr %arrayidx, align 4
   br label %return

 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    lock btcl $4, (%rdi)
 ; X64-NEXT:    setb %al
 ; X64-NEXT:    shll $4, %eax
 ; X64-NEXT:    retq
 entry:
   %0 = atomicrmw xor ptr %v, i32 16 monotonic, align 4
-  %and = and i32 %0, 16
+  %and = and i32 16, %0
   ret i32 %and
 }
 
 define i32 @atomic_shl1_xor_32_const_valz(ptr %v) nounwind {
 ; X86-LABEL: atomic_shl1_xor_32_const_valz:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %eax

 ; X64-NEXT:  # %bb.2: # %if.then
 ; X64-NEXT:    movl 16(%rdi), %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB120_1:
 ; X64-NEXT:    movl $123, %eax
 ; X64-NEXT:    retq
 entry:
   %0 = atomicrmw and ptr %v, i32 -17 monotonic, align 4
-  %and = and i32 %0, 16
+  %and = and i32 16, %0
   %tobool.not = icmp eq i32 %and, 0
   br i1 %tobool.not, label %return, label %if.then
 
 if.then:                                          ; preds = %entry
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 4
   %1 = load i32, ptr %arrayidx, align 4
   br label %return
 
if.then:                                          ; preds = %entry
   %arrayidx = getelementptr inbounds i32, ptr %v, i64 4
   %1 = load i32, ptr %arrayidx, align 4
   br label %return
 
 return:                                           ; preds = %entry, %if.then
   %retval.0 = phi i32 [ %1, %if.then ], [ 123, %entry ]
   ret i32 %retval.0
 }
+
+; This IR isn't really ever expected. This test is just make sure we don't crash.
+define i32 @atomic_xor_dead_and(ptr %v, i32 %c) nounwind {
+; X86-LABEL: atomic_xor_dead_and:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andb $7, %cl
+; X86-NEXT:    movl $1, %esi
+; X86-NEXT:    shll %cl, %esi
+; X86-NEXT:    movl (%edx), %eax
+; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:  .LBB122_1: # %atomicrmw.start
+; X86-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    xorl %esi, %ecx
+; X86-NEXT:    lock cmpxchgl %ecx, (%edx)
+; X86-NEXT:    jne .LBB122_1
+; X86-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-NEXT:    andl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+;
+; X64-LABEL: atomic_xor_dead_and:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    andb $7, %cl
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %edx
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB122_1: # %atomicrmw.start
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    xorl %edx, %ecx
+; X64-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; X64-NEXT:    jne .LBB122_1
+; X64-NEXT:  # %bb.2: # %atomicrmw.end
+; X64-NEXT:    andl %edx, %eax
+; X64-NEXT:    retq
+entry:
+  %0 = and i32 %c, 7
+  %shl = shl nuw nsw i32 1, %0
+  %1 = atomicrmw xor ptr %v, i32 %shl monotonic, align 4
+  %and = and i32 %1, %1
+  %and1 = and i32 %and, %shl
+  ret i32 %and1
+}