diff --git a/mlir/lib/Analysis/AffineStructures.cpp b/mlir/lib/Analysis/AffineStructures.cpp
--- a/mlir/lib/Analysis/AffineStructures.cpp
+++ b/mlir/lib/Analysis/AffineStructures.cpp
@@ -731,8 +731,11 @@
   SmallVector<Value, 4> operands = ifOp.getOperands();
   cst.setIdValues(0, cst.getNumDimAndSymbolIds(), operands);
 
-  // Merge the constraints from ifOp to the current domain.
+  // Merge the constraints from ifOp to the current domain. We need first merge
+  // and align the IDs from both constraints, and then append the constraints
+  // from the ifOp into the current one.
   mergeAndAlignIdsWithOther(0, &cst);
+  append(cst);
 }
 
 // Searches for a constraint with a non-zero coefficient at 'colIdx' in
diff --git a/mlir/test/Transforms/memref-dependence-check.mlir b/mlir/test/Transforms/memref-dependence-check.mlir
--- a/mlir/test/Transforms/memref-dependence-check.mlir
+++ b/mlir/test/Transforms/memref-dependence-check.mlir
@@ -17,7 +17,7 @@
       affine.store %cf7, %m[%i0] : memref<10xf32>
       // expected-remark@above {{dependence from 0 to 0 at depth 1 = false}}
       // expected-remark@above {{dependence from 0 to 0 at depth 2 = false}}
-      // expected-remark@above {{dependence from 0 to 1 at depth 1 = true}}
+      // expected-remark@above {{dependence from 0 to 1 at depth 1 = false}}
     }
   }
   affine.for %i1 = 0 to 10 {
@@ -1044,7 +1044,7 @@
   %N = dim %0, %c0 : memref<101xf32>
   %cf7 = constant 7.0 : f32
 
-  affine.for %i0 = 0 to 100 {
+  affine.for %i0 = 0 to 101 {
     affine.if #set1(%i0)[%N] {
       %1 = affine.load %0[%i0] : memref<101xf32>
       // expected-remark@above {{dependence from 0 to 0 at depth 1 = false}}