Index: llvm/trunk/include/llvm/IR/PatternMatch.h
===================================================================
--- llvm/trunk/include/llvm/IR/PatternMatch.h
+++ llvm/trunk/include/llvm/IR/PatternMatch.h
@@ -1463,6 +1463,20 @@
       if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS))
         return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
 
+    // Match special-case for increment-by-1.
+    if (Pred == ICmpInst::ICMP_EQ) {
+      // (a + 1) == 0
+      // (1 + a) == 0
+      if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) &&
+          (m_One().match(AddLHS) || m_One().match(AddRHS)))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
+      // 0 == (a + 1)
+      // 0 == (1 + a)
+      if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) &&
+          (m_One().match(AddLHS) || m_One().match(AddRHS)))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
+    }
+
     return false;
   }
 };
Index: llvm/trunk/test/CodeGen/Hexagon/swp-epilog-phi5.ll
===================================================================
--- llvm/trunk/test/CodeGen/Hexagon/swp-epilog-phi5.ll
+++ llvm/trunk/test/CodeGen/Hexagon/swp-epilog-phi5.ll
@@ -9,8 +9,8 @@
 
 ; CHECK: loop0
 ; CHECK: [[REG0:r([0-9]+)]] += mpyi
-; CHECK-NOT: r{{[0-9]+}} += add([[REG0]],#8)
-; CHECK: endloop1
+; CHECK: [[REG2:r([0-9]+)]] = add([[REG1:r([0-9]+)]],add([[REG0]],#8
+; CHECK: endloop0
 
 %s.0 = type { %s.1*, %s.4*, %s.7*, i8*, i8, i32, %s.8*, i32, i32, i32, i8, i8, i32, i32, double, i8, i8, i8, i8, i8, i8, i8, i8, i32, i8, i8, i8, i32, i32, i32, i32, i32, i32, i8**, i32, i32, i32, i32, i32, [64 x i32]*, [4 x %s.9*], [4 x %s.10*], [4 x %s.10*], i32, %s.23*, i8, i8, [16 x i8], [16 x i8], [16 x i8], i32, i8, i8, i8, i8, i16, i16, i8, i8, i8, %s.11*, i32, i32, i32, i32, i8*, i32, [4 x %s.23*], i32, i32, i32, [10 x i32], i32, i32, i32, i32, i32, %s.12*, %s.13*, %s.14*, %s.15*, %s.16*, %s.17*, %s.18*, %s.19*, %s.20*, %s.21*, %s.22* }
 %s.1 = type { void (%s.2*)*, void (%s.2*, i32)*, void (%s.2*)*, void (%s.2*, i8*)*, void (%s.2*)*, i32, %s.3, i32, i32, i8**, i32, i8**, i32, i32 }
Index: llvm/trunk/test/CodeGen/X86/copy-eflags.ll
===================================================================
--- llvm/trunk/test/CodeGen/X86/copy-eflags.ll
+++ llvm/trunk/test/CodeGen/X86/copy-eflags.ll
@@ -102,13 +102,13 @@
 ; X32-NEXT:    calll external
 ; X32-NEXT:    addl $4, %esp
 ; X32-NEXT:    testb %bl, %bl
-; X32-NEXT:    je .LBB1_1
-; X32-NEXT:  # %bb.2: # %else
-; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    jne .LBB1_2
+; X32-NEXT:  # %bb.1: # %then
+; X32-NEXT:    movl $64, %eax
 ; X32-NEXT:    popl %ebx
 ; X32-NEXT:    retl
-; X32-NEXT:  .LBB1_1: # %then
-; X32-NEXT:    movl $64, %eax
+; X32-NEXT:  .LBB1_2: # %else
+; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    popl %ebx
 ; X32-NEXT:    retl
 ;
@@ -120,13 +120,13 @@
 ; X64-NEXT:    movl $42, %edi
 ; X64-NEXT:    callq external
 ; X64-NEXT:    testb %bl, %bl
-; X64-NEXT:    je .LBB1_1
-; X64-NEXT:  # %bb.2: # %else
-; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    jne .LBB1_2
+; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    movl $64, %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB1_1: # %then
-; X64-NEXT:    movl $64, %eax
+; X64-NEXT:  .LBB1_2: # %else
+; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
 entry:
Index: llvm/trunk/test/CodeGen/X86/rd-mod-wr-eflags.ll
===================================================================
--- llvm/trunk/test/CodeGen/X86/rd-mod-wr-eflags.ll
+++ llvm/trunk/test/CodeGen/X86/rd-mod-wr-eflags.ll
@@ -177,11 +177,11 @@
 ; CHECK-NEXT:    jne .LBB4_4
 ; CHECK-NEXT:  # %bb.3: # %if.end2
 ; CHECK-NEXT:    incb 14(%rdi)
-; CHECK-NEXT:    je .LBB4_5
+; CHECK-NEXT:    jne .LBB4_4
+; CHECK-NEXT:  # %bb.5: # %if.end4
+; CHECK-NEXT:    jmp other # TAILCALL
 ; CHECK-NEXT:  .LBB4_4: # %return
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB4_5: # %if.end4
-; CHECK-NEXT:    jmp other # TAILCALL
 entry:
   %s64 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 0
   %0 = load i64, i64* %s64, align 8
Index: llvm/trunk/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
===================================================================
--- llvm/trunk/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
+++ llvm/trunk/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
@@ -105,10 +105,11 @@
 
 define i1 @uaddo_i64_increment(i64 %x, i64* %p) {
 ; CHECK-LABEL: @uaddo_i64_increment(
-; CHECK-NEXT:    [[A:%.*]] = add i64 [[X:%.*]], 1
-; CHECK-NEXT:    [[OV:%.*]] = icmp eq i64 [[A]], 0
-; CHECK-NEXT:    store i64 [[A]], i64* [[P:%.*]]
-; CHECK-NEXT:    ret i1 [[OV]]
+; CHECK-NEXT:    [[UADD_OVERFLOW:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT:    [[UADD:%.*]] = extractvalue { i64, i1 } [[UADD_OVERFLOW]], 0
+; CHECK-NEXT:    [[OVERFLOW:%.*]] = extractvalue { i64, i1 } [[UADD_OVERFLOW]], 1
+; CHECK-NEXT:    store i64 [[UADD]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OVERFLOW]]
 ;
   %a = add i64 %x, 1
   %ov = icmp eq i64 %a, 0
@@ -118,10 +119,11 @@
 
 define i1 @uaddo_i8_increment_noncanonical_1(i8 %x, i8* %p) {
 ; CHECK-LABEL: @uaddo_i8_increment_noncanonical_1(
-; CHECK-NEXT:    [[A:%.*]] = add i8 1, [[X:%.*]]
-; CHECK-NEXT:    [[OV:%.*]] = icmp eq i8 [[A]], 0
-; CHECK-NEXT:    store i8 [[A]], i8* [[P:%.*]]
-; CHECK-NEXT:    ret i1 [[OV]]
+; CHECK-NEXT:    [[UADD_OVERFLOW:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 1, i8 [[X:%.*]])
+; CHECK-NEXT:    [[UADD:%.*]] = extractvalue { i8, i1 } [[UADD_OVERFLOW]], 0
+; CHECK-NEXT:    [[OVERFLOW:%.*]] = extractvalue { i8, i1 } [[UADD_OVERFLOW]], 1
+; CHECK-NEXT:    store i8 [[UADD]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OVERFLOW]]
 ;
   %a = add i8 1, %x        ; commute
   %ov = icmp eq i8 %a, 0
@@ -131,10 +133,11 @@
 
 define i1 @uaddo_i32_increment_noncanonical_2(i32 %x, i32* %p) {
 ; CHECK-LABEL: @uaddo_i32_increment_noncanonical_2(
-; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[OV:%.*]] = icmp eq i32 0, [[A]]
-; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]]
-; CHECK-NEXT:    ret i1 [[OV]]
+; CHECK-NEXT:    [[UADD_OVERFLOW:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 1)
+; CHECK-NEXT:    [[UADD:%.*]] = extractvalue { i32, i1 } [[UADD_OVERFLOW]], 0
+; CHECK-NEXT:    [[OVERFLOW:%.*]] = extractvalue { i32, i1 } [[UADD_OVERFLOW]], 1
+; CHECK-NEXT:    store i32 [[UADD]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OVERFLOW]]
 ;
   %a = add i32 %x, 1
   %ov = icmp eq i32 0, %a   ; commute
@@ -144,10 +147,11 @@
 
 define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, i16* %p) {
 ; CHECK-LABEL: @uaddo_i16_increment_noncanonical_3(
-; CHECK-NEXT:    [[A:%.*]] = add i16 1, [[X:%.*]]
-; CHECK-NEXT:    [[OV:%.*]] = icmp eq i16 0, [[A]]
-; CHECK-NEXT:    store i16 [[A]], i16* [[P:%.*]]
-; CHECK-NEXT:    ret i1 [[OV]]
+; CHECK-NEXT:    [[UADD_OVERFLOW:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 1, i16 [[X:%.*]])
+; CHECK-NEXT:    [[UADD:%.*]] = extractvalue { i16, i1 } [[UADD_OVERFLOW]], 0
+; CHECK-NEXT:    [[OVERFLOW:%.*]] = extractvalue { i16, i1 } [[UADD_OVERFLOW]], 1
+; CHECK-NEXT:    store i16 [[UADD]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OVERFLOW]]
 ;
   %a = add i16 1, %x        ; commute
   %ov = icmp eq i16 0, %a   ; commute
@@ -159,10 +163,11 @@
 
 define i1 @uaddo_i42_increment_illegal_type(i42 %x, i42* %p) {
 ; CHECK-LABEL: @uaddo_i42_increment_illegal_type(
-; CHECK-NEXT:    [[A:%.*]] = add i42 [[X:%.*]], 1
-; CHECK-NEXT:    [[OV:%.*]] = icmp eq i42 [[A]], 0
-; CHECK-NEXT:    store i42 [[A]], i42* [[P:%.*]]
-; CHECK-NEXT:    ret i1 [[OV]]
+; CHECK-NEXT:    [[UADD_OVERFLOW:%.*]] = call { i42, i1 } @llvm.uadd.with.overflow.i42(i42 [[X:%.*]], i42 1)
+; CHECK-NEXT:    [[UADD:%.*]] = extractvalue { i42, i1 } [[UADD_OVERFLOW]], 0
+; CHECK-NEXT:    [[OVERFLOW:%.*]] = extractvalue { i42, i1 } [[UADD_OVERFLOW]], 1
+; CHECK-NEXT:    store i42 [[UADD]], i42* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OVERFLOW]]
 ;
   %a = add i42 %x, 1
   %ov = icmp eq i42 %a, 0