Index: lib/Target/AMDGPU/GCNDPPCombine.cpp
===================================================================
--- lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -12,26 +12,29 @@
 //
 // $old = ...
 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
-//                            dpp_controls..., $bound_ctrl
+//                            dpp_controls, $row_mask, $bank_mask, $bound_ctrl
 // $res = VALU $dpp_value, ...
 //
 // to
 //
-// $res = VALU_DPP $folded_old, $vgpr_to_be_read_from_other_lane, ...,
-//                 dpp_controls..., $folded_bound_ctrl
+// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, ...,
+//                 dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
 //
 // Combining rules :
 //
-// $bound_ctrl is DPP_BOUND_ZERO, $old is any
-// $bound_ctrl is DPP_BOUND_OFF, $old is 0
+// if $row_mask and $bank_mask are all enabled (0xF) and
+//   $bound_ctrl==DPP_BOUND_ZERO and $old==any or
+//   $bound_ctrl==DPP_BOUND_OFF and $old==0
+// -> $combined_old = undef,
+//    $combined_bound_ctrl = DPP_BOUND_ZERO
 //
-// ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_ZERO
-// $bound_ctrl is DPP_BOUND_OFF, $old is undef
+// Otherwise:
 //
-// ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_OFF
-// $bound_ctrl is DPP_BOUND_OFF, $old is foldable
+// if $old==undef or $old==identity value for the VALU op
+// -> $combined_old = undef/folded identity value,
+//    $combined_bound_ctrl = DPP_BOUND_OFF
 //
-// ->$folded_old = folded value, $folded_bound_ctrl = DPP_BOUND_OFF
+// Othervise cancel.
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
@@ -136,6 +139,10 @@
   if (!Def)
     return nullptr;
 
+  // def should be in the same BB
+  if (Def->getParent() != OldOpnd.getParent()->getParent())
+    return &OldOpnd;
+
   switch(Def->getOpcode()) {
   default: break;
   case AMDGPU::IMPLICIT_DEF:
@@ -199,6 +206,7 @@
       break;
     }
     DPPInst.add(*Src0);
+    DPPInst->getOperand(NumOperands).setIsKill(false);
     ++NumOperands;
 
     if (auto *Mod1 = TII->getNamedOperand(OrigMI,
@@ -249,8 +257,18 @@
   assert(OldOpndValue.isImm());
   switch (OrigMI.getOpcode()) {
   default: break;
+  case AMDGPU::V_ADD_U32_e32:
+  case AMDGPU::V_ADD_I32_e32:
+  case AMDGPU::V_AND_B32_e32:
+  case AMDGPU::V_SUBREV_U32_e32:
+  case AMDGPU::V_SUBREV_I32_e32:
+    if (OldOpndValue.getImm() == 0)
+      return OldOpndVGPR;
+    break;
+  case AMDGPU::V_OR_B32_e32:
   case AMDGPU::V_MAX_U32_e32:
-    if (OldOpndValue.getImm() == std::numeric_limits<uint32_t>::max())
+    if (static_cast<uint32_t>(OldOpndValue.getImm()) ==
+        std::numeric_limits<uint32_t>::max())
       return OldOpndVGPR;
     break;
   case AMDGPU::V_MAX_I32_e32:
@@ -274,17 +292,6 @@
   return RegSubRegPair();
 }
 
-// Cases to combine:
-//  $bound_ctrl is DPP_BOUND_ZERO, $old is any
-//  $bound_ctrl is DPP_BOUND_OFF, $old is 0
-//  -> $old = undef, $bound_ctrl = DPP_BOUND_ZERO
-
-//  $bound_ctrl is DPP_BOUND_OFF, $old is undef
-//  -> $old = undef, $bound_ctrl = DPP_BOUND_OFF
-
-//  $bound_ctrl is DPP_BOUND_OFF, $old is foldable
-//  -> $old = folded value, $bound_ctrl = DPP_BOUND_OFF
-
 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
                                            MachineInstr &MovMI,
                                            RegSubRegPair OldOpndVGPR,
@@ -316,9 +323,17 @@
 
 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp);
+
+  auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
+  assert(RowMaskOpnd && RowMaskOpnd->isImm());
+  auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
+  assert(BankMaskOpnd && BankMaskOpnd->isImm());
+  const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
+                            BankMaskOpnd->getImm() == 0xF;
+
   auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);
   assert(BCZOpnd && BCZOpnd->isImm());
-  bool BoundCtrlZero = 0 != BCZOpnd->getImm();
+  bool BoundCtrlZero = MaskAllLanes && 0 != BCZOpnd->getImm();
 
   LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
 
@@ -336,7 +351,7 @@
         LLVM_DEBUG(dbgs() << "  failed: old operand isn't an imm or undef\n");
         return false;
       }
-      if (OldOpndValue->getImm() == 0) {
+      if (MaskAllLanes && OldOpndValue->getImm() == 0) {
         OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef
         OldOpndValue = nullptr;
         BoundCtrlZero = true;
@@ -367,6 +382,11 @@
     Rollback = true;
 
     auto &OrigMI = *Use.getParent();
+    if (OrigMI.getParent() != MovMI.getParent()) {
+      LLVM_DEBUG(dbgs() << "  failed: mov and use are in different BBs\n");
+      break;
+    }
+
     auto OrigOp = OrigMI.getOpcode();
     if (TII->isVOP3(OrigOp)) {
       if (!TII->hasVALU32BitEncoding(OrigOp)) {
Index: test/CodeGen/AMDGPU/dpp_combine.ll
===================================================================
--- test/CodeGen/AMDGPU/dpp_combine.ll
+++ test/CodeGen/AMDGPU/dpp_combine.ll
@@ -2,20 +2,20 @@
 
 ; VOP2 with literal cannot be combined
 ; CHECK-LABEL: {{^}}dpp_combine_i32_literal:
-; CHECK: v_mov_b32_dpp [[OLD:v[0-9]+]], {{v[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x2 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_mov_b32_dpp [[OLD:v[0-9]+]], {{v[0-9]+}} quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 ; CHECK: v_add_u32_e32 {{v[0-9]+}}, vcc, 42, [[OLD]]
 define amdgpu_kernel void @dpp_combine_i32_literal(i32 addrspace(1)* %out, i32 %in) {
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 2, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0
   %res = add nsw i32 %dpp, 42
   store i32 %res, i32 addrspace(1)* %out
   ret void
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_i32_bz:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 define amdgpu_kernel void @dpp_combine_i32_bz(i32 addrspace(1)* %out, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0
   %res = add nsw i32 %dpp, %x
   store i32 %res, i32 addrspace(1)* %out
   ret void
@@ -32,15 +32,36 @@
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_i32_boff_0:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 define amdgpu_kernel void @dpp_combine_i32_boff_0(i32 addrspace(1)* %out, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 15, i32 15, i1 0) #0
   %res = add nsw i32 %dpp, %x
   store i32 %res, i32 addrspace(1)* %out
   ret void
 }
 
+; CHECK-LABEL: {{^}}dpp_combine_i32_boff_or:
+; CHECK: v_mov_b32_e32 [[OLD:v[0-9]+]], -1
+; CHECK: v_or_b32_dpp [[OLD]], {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+define amdgpu_kernel void @dpp_combine_i32_boff_or(i32 addrspace(1)* %out, i32 %in) {
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 4294967295, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
+  %res = or i32 %dpp, %x
+  store i32 %res, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}dpp_combine_i32_boff_and:
+; CHECK: v_and_b32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+define amdgpu_kernel void @dpp_combine_i32_boff_and(i32 addrspace(1)* %out, i32 %in) {
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
+  %res = and i32 %dpp, %x
+  store i32 %res, i32 addrspace(1)* %out
+  ret void
+}
+
 ; CHECK-LABEL: {{^}}dpp_combine_i32_boff_max:
 ; CHECK: v_bfrev_b32_e32 [[OLD:v[0-9]+]], -2
 ; CHECK: v_max_i32_dpp [[OLD]], {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
@@ -81,21 +102,21 @@
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_i32_commute:
-; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[2,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[2,0,0,0] row_mask:0x1 bank_mask:0x1
 define amdgpu_kernel void @dpp_combine_i32_commute(i32 addrspace(1)* %out, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 2, i32 1, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 2, i32 1, i32 1, i1 1) #0
   %res = sub nsw i32 %x, %dpp
   store i32 %res, i32 addrspace(1)* %out
   ret void
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_f32:
-; CHECK: v_add_f32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0  quad_perm:[3,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_add_f32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0  quad_perm:[3,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 define amdgpu_kernel void @dpp_combine_f32(i32 addrspace(1)* %out, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
 
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 3, i32 1, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 3, i32 15, i32 15, i1 1) #0
   %dpp.f32 = bitcast i32 %dpp to float
   %x.f32 = bitcast i32 %x to float
   %res.f32 = fadd float %x.f32, %dpp.f32
@@ -105,11 +126,11 @@
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_test_f32_mods:
-; CHECK: v_mul_f32_dpp {{v[0-9]+}}, |{{v[0-9]+}}|, -v0  quad_perm:[0,1,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_mul_f32_dpp {{v[0-9]+}}, |{{v[0-9]+}}|, -v0  quad_perm:[0,1,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 define amdgpu_kernel void @dpp_combine_test_f32_mods(i32 addrspace(1)* %out, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
 
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 4, i32 1, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 4, i32 15, i32 15, i1 1) #0
 
   %x.f32 = bitcast i32 %x to float
   %x.f32.neg = fsub float -0.000000e+00, %x.f32
@@ -126,11 +147,11 @@
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_mac:
-; CHECK: v_mac_f32_dpp v0, {{v[0-9]+}}, v1  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_mac_f32_dpp v0, {{v[0-9]+}}, v1  quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 define amdgpu_kernel void @dpp_combine_mac(float addrspace(1)* %out, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
   %y = tail call i32 @llvm.amdgcn.workitem.id.y()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0
   %dpp.f32 = bitcast i32 %dpp to float
   %x.f32 = bitcast i32 %x to float
   %y.f32 = bitcast i32 %y to float
@@ -142,29 +163,26 @@
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_sequence:
-define amdgpu_kernel void @dpp_combine_sequence(i32 addrspace(1)* %out, i32 %in, i1 %cmp) {
+define amdgpu_kernel void @dpp_combine_sequence(i32 addrspace(1)* %out1, i32 addrspace(1)* %out2, i32 %in) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
-  br i1 %cmp, label %bb1, label %bb2
-bb1:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0
+
+; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
   %resadd = add nsw i32 %dpp, %x
-  br label %bb3
-bb2:
-; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+
+; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
   %ressub = sub nsw i32 %x, %dpp
-  br label %bb3
-bb3:
-  %res = phi i32 [%resadd, %bb1], [%ressub, %bb2]
-  store i32 %res, i32 addrspace(1)* %out
+
+  store i32 %resadd, i32 addrspace(1)* %out1
+  store i32 %ressub, i32 addrspace(1)* %out2
   ret void
 }
 
 ; CHECK-LABEL: {{^}}dpp_combine_sequence_negative:
-; CHECK: v_mov_b32_dpp v1, v1  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; CHECK: v_mov_b32_dpp v1, v1  quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0
 define amdgpu_kernel void @dpp_combine_sequence_negative(i32 addrspace(1)* %out, i32 %in, i1 %cmp) {
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
+  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0
   br i1 %cmp, label %bb1, label %bb2
 bb1:
   %resadd = add nsw i32 %dpp, %x
Index: test/CodeGen/AMDGPU/dpp_combine_subregs.mir
===================================================================
--- test/CodeGen/AMDGPU/dpp_combine_subregs.mir
+++ test/CodeGen/AMDGPU/dpp_combine_subregs.mir
@@ -36,8 +36,7 @@
 ...
 
 # CHECK-LABEL: name: add_old_subreg
-# CHECK: [[OLD:\%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-# CHECK: %5:vgpr_32 = V_ADD_U32_dpp [[OLD]], %1, %0.sub1, 1, 1, 1, 1, implicit $exec
+# CHECK: %5:vgpr_32 = V_ADD_U32_dpp %3.sub1, %1, %0.sub1, 1, 1, 1, 0, implicit $exec
 
 name:            add_old_subreg
 tracksRegLiveness: true
@@ -93,10 +92,10 @@
 ...
 
 # CHECK-LABEL: name: add_f32_e64
-# CHECK: %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
+# CHECK: %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
 # CHECK: %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec
-# CHECK: %6:vgpr_32 = V_ADD_F32_dpp %2, 0, %1, 0, %0, 1, 1, 1, 1, implicit $exec
-# CHECK: %7:vgpr_32 = V_ADD_F32_dpp %2, 1, %1, 2, %0, 1, 1, 1, 1, implicit $exec
+# CHECK: %6:vgpr_32 = V_ADD_F32_dpp %2, 0, %1, 0, %0, 1, 15, 15, 1, implicit $exec
+# CHECK: %7:vgpr_32 = V_ADD_F32_dpp %2, 1, %1, 2, %0, 1, 15, 15, 1, implicit $exec
 # CHECK: %9:vgpr_32 = V_ADD_F32_e64 4, %8, 8, %0, 0, 0, implicit $exec
 
 name:            add_f32_e64
@@ -123,12 +122,12 @@
     %0:vgpr_32 = COPY $vgpr0
     %1:vgpr_32 = COPY $vgpr1
     %2:vgpr_32 = IMPLICIT_DEF
-    %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
 
     ; this shouldn't be combined as omod is set
     %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec
 
-    %5:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
+    %5:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
 
     ; this should be combined as all modifiers are default
     %6:vgpr_32 = V_ADD_F32_e64 0, %5, 0, %0, 0, 0, implicit $exec
@@ -136,7 +135,7 @@
     ; this should be combined as modifiers other than abs|neg are default
     %7:vgpr_32 = V_ADD_F32_e64 1, %5, 2, %0, 0, 0, implicit $exec
 
-    %8:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
+    %8:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
 
     ; this shouldn't be combined as modifiers aren't abs|neg
     %9:vgpr_32 = V_ADD_F32_e64 4, %8, 8, %0, 0, 0, implicit $exec