Index: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
===================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2965,12 +2965,12 @@
     case ISD::ZERO_EXTEND:
       LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
                         DAG.getValueType(AtomicType));
-      RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
+      RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
       ExtRes = LHS;
       break;
     case ISD::ANY_EXTEND:
       LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
-      RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
+      RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
       break;
     default:
       llvm_unreachable("Invalid atomic op extension");
Index: llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll
===================================================================
--- llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll
+++ llvm/trunk/test/CodeGen/AArch64/atomic-ops-lse.ll
@@ -629,12 +629,27 @@
 
 ; CHECK-NOT: dmb
 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
-; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
+; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
+; CHECK-NEXT: casab w0, w1, [x[[ADDR]]]
+; CHECK-NEXT: ret
+
+   ret i8 %old
+}
+
+define i1 @test_atomic_cmpxchg_i8_1(i8 %wanted, i8 %new) nounwind {
+; CHECK-LABEL: test_atomic_cmpxchg_i8_1:
+   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
+   %success = extractvalue { i8, i1 } %pair, 1
 
-; CHECK: casab w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]]
 ; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
+; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
 
-   ret i8 %old
+; CHECK: casab w[[NEW:[0-9]+]], w1, [x[[ADDR]]]
+; CHECK-NEXT: cmp w[[NEW]], w0, uxtb
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+   ret i1 %success
 }
 
 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
@@ -644,12 +659,28 @@
 
 ; CHECK-NOT: dmb
 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
-; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
+; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
+; CHECK-NEXT: casah w0, w1, [x[[ADDR]]]
+; CHECK-NEXT: ret
+
+   ret i16 %old
+}
+
+define i1 @test_atomic_cmpxchg_i16_1(i16 %wanted, i16 %new) nounwind {
+; CHECK-LABEL: test_atomic_cmpxchg_i16_1:
+   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire
+   %success = extractvalue { i16, i1 } %pair, 1
 
-; CHECK: casah w0, w1, [x[[ADDR]]]
 ; CHECK-NOT: dmb
+; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
+; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
 
-   ret i16 %old
+; CHECK: casah w[[NEW:[0-9]+]], w1, [x[[ADDR]]]
+; CHECK-NEXT: cmp w[[NEW]], w0, uxth
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+
+   ret i1 %success
 }
 
 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
Index: llvm/trunk/test/CodeGen/ARM/atomic-cmpxchg.ll
===================================================================
--- llvm/trunk/test/CodeGen/ARM/atomic-cmpxchg.ll
+++ llvm/trunk/test/CodeGen/ARM/atomic-cmpxchg.ll
@@ -49,9 +49,10 @@
 ; CHECK-THUMBV6:       mov [[EXPECTED:r[0-9]+]], r1
 ; CHECK-THUMBV6-NEXT:  bl __sync_val_compare_and_swap_1
 ; CHECK-THUMBV6-NEXT:  mov [[RES:r[0-9]+]], r0
+; CHECK-THUMBV6-NEXT:  uxtb [[EXPECTED_ZEXT:r[0-9]+]], [[EXPECTED]]
 ; CHECK-THUMBV6-NEXT:  movs r0, #1
 ; CHECK-THUMBV6-NEXT:  movs [[ZERO:r[0-9]+]], #0
-; CHECK-THUMBV6-NEXT:  cmp [[RES]], [[EXPECTED]]
+; CHECK-THUMBV6-NEXT:  cmp [[RES]], [[EXPECTED_ZEXT]]
 ; CHECK-THUMBV6-NEXT:  beq [[END:.LBB[0-9_]+]]
 ; CHECK-THUMBV6-NEXT:  mov r0, [[ZERO]]
 ; CHECK-THUMBV6-NEXT: [[END]]:
Index: llvm/trunk/test/CodeGen/ARM/cmpxchg-O0.ll
===================================================================
--- llvm/trunk/test/CodeGen/ARM/cmpxchg-O0.ll
+++ llvm/trunk/test/CodeGen/ARM/cmpxchg-O0.ll
@@ -17,7 +17,8 @@
 ; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
 ; CHECK:     bne [[RETRY]]
 ; CHECK: [[DONE]]:
-; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED]]
+; CHECK:     uxtb [[DESIRED_ZEXT:r[0-9]+]], [[DESIRED]]
+; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED_ZEXT]]
 ; CHECK:     {{moveq|movweq}} {{r[0-9]+}}, #1
 ; CHECK:     dmb ish
   %res = cmpxchg i8* %addr, i8 %desired, i8 %new seq_cst monotonic
@@ -36,7 +37,8 @@
 ; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
 ; CHECK:     bne [[RETRY]]
 ; CHECK: [[DONE]]:
-; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED]]
+; CHECK:     uxth [[DESIRED_ZEXT:r[0-9]+]], [[DESIRED]]
+; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED_ZEXT]]
 ; CHECK:     {{moveq|movweq}} {{r[0-9]+}}, #1
 ; CHECK:     dmb ish
   %res = cmpxchg i16* %addr, i16 %desired, i16 %new seq_cst monotonic
Index: llvm/trunk/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
===================================================================
--- llvm/trunk/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
+++ llvm/trunk/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
@@ -26,9 +26,7 @@
 ; CHECK:  .LBB0_3: # %L.entry
 ; CHECK:    sthcx. 3, 0, 5
 ; CHECK:  .LBB0_4: # %L.entry
-; Once D41798 lands, this should be the check:
-; FIXME:    cmplwi 3, 33059
-; CHECK:    cmpwi 3, -32477
+; CHECK:    cmplwi 3, 33059
 ; CHECK:    lwsync
 ; CHECK:    lhz 3, 46(1)
 ; CHECK:    cmplwi 3, 234
@@ -64,9 +62,7 @@
 ; CHECK-P7:  .LBB0_4: # %L.entry
 ; CHECK-P7:    srw 3, 6, 3
 ; CHECK-P7:    lwsync
-; Once D41798 lands, this should be the check:
-; FIXME:    cmplwi 3, 33059
-; CHECK-P7:    cmpwi 3, -32477
+; CHECK-P7:    cmplwi 3, 33059
 ; CHECK-P7:    lhz 3, 46(1)
 ; CHECK-P7:    cmplwi 3, 234
 L.entry: