[DAG] More aggressively Inline TokenFactors

[niravd]

Summary

Now the anti-aliasing in SelectionDAG is on by default, fold in TokenFactors of
TokenFactors with multiple uses for TFs with relatively small number of chains.

This marginally improves DAG analysis.

[hfinkel]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1599

Are these lines too long?

1599

Please add a cl::opt instead of embedding 32 in the code.

1599

Also, please add a comment that explains the limit here. The issue is that, if the TF operand is a large TF, then doing this will "duplicate" it (by folding it into other TFs) many times. In that light, it seems like Ops.size()*Ops.use_size() is really the relevant thing to bound.

test/CodeGen/PowerPC/complex-return.ll

12–13

Why is this changing?

[niravd]

• [DAG] Prevent Stale nodes from entering worklist

[efriedma]

I'm vaguely worried this is going to hurt compile-time without really being helpful in general; you can't really do much useful with multiple overlapping TokenFactors, and blindly inlining TokenFactors is going to hit the limit quickly on large basic blocks. Is it possible some more targeted fix could handle the interesting cases?

[niravd]

Remove erroneous update and address hal's comments

[niravd]

I've been planning this patch waiting for D14834 to land. Looking closer I can no longer find any cases where this improves code quality, so dropping it seems like the thing to do.

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

 
 #ifndef NDEBUG
   static cl::opt<std::string>
     CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
                cl::desc("Only use DAG-combiner alias analysis in this"
                         " function"));
 #endif
 
+  static cl::opt<bool>
+      MaxParallelChains("combiner-max-tf-size", cl::Hidden, cl::init(64),
+                        cl::desc("Maximum number of parallel chains in "
+                                 "TokenFactor to allow for inlining"));
+
   /// Hidden option to stress test load slicing, i.e., when this option
   /// is enabled, load slicing bypasses most of its profitability guards.
   static cl::opt<bool>
   StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
                     cl::desc("Bypass the profitability model of load "
                              "slicing"),
                     cl::init(false));
 
for (const SDValue &Op : TF->op_values()) {
       switch (Op.getOpcode()) {
       case ISD::EntryToken:
         // Entry tokens don't need to be added to the list. They are
         // redundant.
         Changed = true;
         break;
 
       case ISD::TokenFactor:
-        if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
+        // Inlining multi-use token factors can cause quadratic growth
+        // in node size. Place a bound on the maximum worst-case TF
+        // size when inlining.
+        if ((Op.hasOneUse() ||
+             Op.getNumOperands() + Ops.size() <= MaxParallelChains) &&
+            !is_contained(TFs, Op.getNode())) {
           // Queue up for processing.
           TFs.push_back(Op.getNode());
           // Clean up in case the token factor is removed.
           AddToWorklist(Op.getNode());
           Changed = true;
           break;
         }
         LLVM_FALLTHROUGH;

test/CodeGen/AArch64/arm64-abi_align.ll

; FAST: add {{w[0-9]+}}, w[[C]], w[[B]]
   %add7 = add i32 %add4, %conv6
   ret i32 %add7
 }
 
 ; For s1, we allocate a 22-byte space, pass its address via x1
 define i32 @caller42() #3 {
 entry:
 ; CHECK-LABEL: caller42
-; CHECK: str {{x[0-9]+}}, [sp, #48]
-; CHECK: str {{q[0-9]+}}, [sp, #32]
-; CHECK: str {{x[0-9]+}}, [sp, #16]
-; CHECK: str {{q[0-9]+}}, [sp]
+; CHECK-DAG: str {{x[0-9]+}}, [sp, #48]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #32]
+; CHECK-DAG: str {{x[0-9]+}}, [sp, #16]
+; CHECK-DAG: str {{q[0-9]+}}, [sp]
 ; CHECK: add x1, sp, #32
 ; CHECK: mov x2, sp
 ; Space for s1 is allocated at sp+32
 ; Space for s2 is allocated at sp
 
 ; FAST-LABEL: caller42
 ; FAST: sub sp, sp, #112
 ; Space for s1 is allocated at fp-24 = sp+72
declare i32 @f42_stack(i32 %i, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6,
                        i32 %i7, i32 %i8, i32 %i9, %struct.s42* nocapture %s1,
                        %struct.s42* nocapture %s2) #2
 
 define i32 @caller42_stack() #3 {
 entry:
 ; CHECK-LABEL: caller42_stack
 ; CHECK: sub sp, sp, #112
 ; CHECK: add x29, sp, #96
-; CHECK: stur {{x[0-9]+}}, [x29, #-16]
-; CHECK: stur {{q[0-9]+}}, [x29, #-32]
-; CHECK: str {{x[0-9]+}}, [sp, #48]
-; CHECK: str {{q[0-9]+}}, [sp, #32]
+; CHECK-DAG: stur {{x[0-9]+}}, [x29, #-16]
+; CHECK-DAG: stur {{q[0-9]+}}, [x29, #-32]
+; CHECK-DAG: str {{x[0-9]+}}, [sp, #48]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #32]
 ; Space for s1 is allocated at x29-32 = sp+64
 ; Space for s2 is allocated at sp+32
 ; CHECK: add x[[B:[0-9]+]], sp, #32
 ; CHECK: str x[[B]], [sp, #16]
 ; CHECK: sub x[[A:[0-9]+]], x29, #32
 ; Address of s1 is passed on stack at sp+8
 ; CHECK: str x[[A]], [sp, #8]
 ; CHECK: mov w[[C:[0-9]+]], #9
; FAST: add {{w[0-9]+}}, w[[C]], w[[B]]
   %add4 = add i32 %add3, %conv
   %add7 = add i32 %add4, %conv6
   ret i32 %add7
 }
 
 define i32 @caller43() #3 {
 entry:
 ; CHECK-LABEL: caller43
-; CHECK: str {{q[0-9]+}}, [sp, #48]
-; CHECK: str {{q[0-9]+}}, [sp, #32]
-; CHECK: str {{q[0-9]+}}, [sp, #16]
-; CHECK: str {{q[0-9]+}}, [sp]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #48]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #32]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #16]
+; CHECK-DAG: str {{q[0-9]+}}, [sp]
 ; CHECK: add x1, sp, #32
 ; CHECK: mov x2, sp
 ; Space for s1 is allocated at sp+32
 ; Space for s2 is allocated at sp
 
 ; FAST-LABEL: caller43
 ; FAST: add x29, sp, #64
 ; Space for s1 is allocated at sp+32
declare i32 @f43_stack(i32 %i, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6,
                        i32 %i7, i32 %i8, i32 %i9, %struct.s43* nocapture %s1,
                        %struct.s43* nocapture %s2) #2
 
 define i32 @caller43_stack() #3 {
 entry:
 ; CHECK-LABEL: caller43_stack
 ; CHECK: sub sp, sp, #112
 ; CHECK: add x29, sp, #96
-; CHECK: stur {{q[0-9]+}}, [x29, #-16]
-; CHECK: stur {{q[0-9]+}}, [x29, #-32]
-; CHECK: str {{q[0-9]+}}, [sp, #48]
-; CHECK: str {{q[0-9]+}}, [sp, #32]
+; CHECK-DAG: stur {{q[0-9]+}}, [x29, #-16]
+; CHECK-DAG: stur {{q[0-9]+}}, [x29, #-32]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #48]
+; CHECK-DAG: str {{q[0-9]+}}, [sp, #32]
 ; Space for s1 is allocated at x29-32 = sp+64
 ; Space for s2 is allocated at sp+32
 ; CHECK: add x[[B:[0-9]+]], sp, #32
 ; CHECK: str x[[B]], [sp, #16]
 ; CHECK: sub x[[A:[0-9]+]], x29, #32
 ; Address of s1 is passed on stack at sp+8
 ; CHECK: str x[[A]], [sp, #8]
 ; CHECK: mov w[[C:[0-9]+]], #9

test/CodeGen/AArch64/arm64-variadic-aapcs.ll

; CHECK: str [[STACK]], [x[[VAR]]]
 
   ret void
 }
 
 ; If there are non-variadic arguments on the stack (here two i64s) then the
 ; __stack field should point just past them.
 define void @test_offsetstack([8 x i64], [2 x i64], [3 x float], ...) {
 ; CHECK-LABEL: test_offsetstack:
-; CHECK: stp {{q[0-9]+}}, {{q[0-9]+}}, [sp, #-80]!
-; CHECK: add [[STACK_TOP:x[0-9]+]], sp, #96
-; CHECK: add x[[VAR:[0-9]+]], {{x[0-9]+}}, :lo12:var
-; CHECK: str [[STACK_TOP]], [x[[VAR]]]
+; CHECK-DAG: stp {{q[0-9]+}}, {{q[0-9]+}}, [sp, #-80]!
+; CHECK-DAG: add [[STACK_TOP:x[0-9]+]], sp, #96
+; CHECK-DAG: add x[[VAR:[0-9]+]], {{x[0-9]+}}, :lo12:var
+; CHECK-DAG: str [[STACK_TOP]], [x[[VAR]]]
 
   %addr = bitcast %va_list* @var to i8*
   call void @llvm.va_start(i8* %addr)
   ret void
 }
 
 declare void @llvm.va_end(i8*)
 

test/CodeGen/BPF/undef.ll

 ; CHECK: r2 = 7
 ; CHECK: *(u8 *)(r10 - 6) = r2
 ; CHECK: r2 = 8
 ; CHECK: *(u8 *)(r10 - 5) = r2
 ; CHECK: r2 = 9
 ; CHECK: *(u8 *)(r10 - 4) = r2
 ; CHECK: r2 = 10
 ; CHECK: *(u8 *)(r10 - 3) = r2
-; CHECK: *(u16 *)(r10 + 24) = r1
-; CHECK: *(u16 *)(r10 + 22) = r1
-; CHECK: *(u16 *)(r10 + 20) = r1
-; CHECK: *(u16 *)(r10 + 18) = r1
-; CHECK: *(u16 *)(r10 + 16) = r1
-; CHECK: *(u16 *)(r10 + 14) = r1
-; CHECK: *(u16 *)(r10 + 12) = r1
-; CHECK: *(u16 *)(r10 + 10) = r1
-; CHECK: *(u16 *)(r10 + 8) = r1
-; CHECK: *(u16 *)(r10 + 6) = r1
-; CHECK: *(u16 *)(r10 - 2) = r1
-; CHECK: *(u16 *)(r10 + 26) = r1
+; CHECK-DAG: *(u16 *)(r10 + 24) = r1
+; CHECK-DAG: *(u16 *)(r10 + 22) = r1
+; CHECK-DAG: *(u16 *)(r10 + 20) = r1
+; CHECK-DAG: *(u16 *)(r10 + 18) = r1
+; CHECK-DAG: *(u16 *)(r10 + 16) = r1
+; CHECK-DAG: *(u16 *)(r10 + 14) = r1
+; CHECK-DAG: *(u16 *)(r10 + 12) = r1
+; CHECK-DAG: *(u16 *)(r10 + 10) = r1
+; CHECK-DAG: *(u16 *)(r10 + 8) = r1
+; CHECK-DAG: *(u16 *)(r10 + 6) = r1
+; CHECK-DAG: *(u16 *)(r10 - 2) = r1
+; CHECK-DAG: *(u16 *)(r10 + 26) = r1
 ; CHECK: r2 = r10
 ; CHECK: r2 += -8
 ; CHECK: r1 = <MCOperand Expr:(routing)>ll
 ; CHECK: call bpf_map_lookup_elem
 ; CHECK: exit
   %key = alloca %struct.routing_key_2, align 1
   %1 = getelementptr inbounds %struct.routing_key_2, %struct.routing_key_2* %key, i64 0, i32 0, i64 0
   store i8 5, i8* %1, align 1

test/CodeGen/MSP430/Inst16mm.ll

  %y = alloca i32, align 2                        ; <i32*> [#uses=1]
  store i16 0, i16* %retval
  %tmp = load i32, i32* %y                             ; <i32> [#uses=1]
  store i32 %tmp, i32* %x
  store i16 0, i16* %retval
  %0 = load i16, i16* %retval                          ; <i16> [#uses=1]
  ret i16 %0
 ; CHECK-LABEL: mov2:
-; CHECK:  mov.w 2(r1), 6(r1)
-; CHECK:  mov.w 0(r1), 4(r1)
+; CHECK-DAG:  mov.w 2(r1), 6(r1)
+; CHECK-DAG:  mov.w 0(r1), 4(r1)
 }

test/CodeGen/PowerPC/complex-return.ll

 ; RUN: llc -verify-machineinstrs -mcpu=ppc64 -O0 < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define { ppc_fp128, ppc_fp128 } @foo() nounwind {
 entry:
   %retval = alloca { ppc_fp128, ppc_fp128 }, align 16
   %x = alloca { ppc_fp128, ppc_fp128 }, align 16
   %real = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 0
   %imag = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 1
-  store ppc_fp128 0xM400C0000000000000000000000000000, ppc_fp128* %real
-  store ppc_fp128 0xMC00547AE147AE1483CA47AE147AE147A, ppc_fp128* %imag
+  store ppc_fp128 0xM400C0000000033300000000888800001, ppc_fp128* %real
+  store ppc_fp128 0xMC00547AE147AE1483CA47AE147AE149A, ppc_fp128* %imag
   %x.realp = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 0
   %x.real = load ppc_fp128, ppc_fp128* %x.realp
   %x.imagp = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 1
   %x.imag = load ppc_fp128, ppc_fp128* %x.imagp
   %real1 = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 0
   %imag2 = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 1
   store ppc_fp128 %x.real, ppc_fp128* %real1
   store ppc_fp128 %x.imag, ppc_fp128* %imag2

test/CodeGen/PowerPC/structsinmem.ll

entry:
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %4, i8* bitcast (%struct.t5* @caller2.p5 to i8*), i64 5, i32 1, i1 false)
   %5 = bitcast %struct.t6* %p6 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %5, i8* bitcast (%struct.t6* @caller2.p6 to i8*), i64 6, i32 1, i1 false)
   %6 = bitcast %struct.t7* %p7 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %6, i8* bitcast (%struct.t7* @caller2.p7 to i8*), i64 7, i32 1, i1 false)
   %call = call i32 @callee2(i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, %struct.t1* byval %p1, %struct.t2* byval %p2, %struct.t3* byval %p3, %struct.t4* byval %p4, %struct.t5* byval %p5, %struct.t6* byval %p6, %struct.t7* byval %p7)
   ret i32 %call
 
-; CHECK: stb {{[0-9]+}}, 119(1)
-; CHECK: sth {{[0-9]+}}, 126(1)
-; CHECK: stb {{[0-9]+}}, 135(1)
-; CHECK: sth {{[0-9]+}}, 133(1)
-; CHECK: stw {{[0-9]+}}, 140(1)
-; CHECK: stb {{[0-9]+}}, 151(1)
-; CHECK: stw {{[0-9]+}}, 147(1)
-; CHECK: sth {{[0-9]+}}, 158(1)
-; CHECK: stw {{[0-9]+}}, 154(1)
-; CHECK: stb {{[0-9]+}}, 167(1)
-; CHECK: sth {{[0-9]+}}, 165(1)
-; CHECK: stw {{[0-9]+}}, 161(1)
+; CHECK-DAG: stb {{[0-9]+}}, 119(1)
+; CHECK-DAG: sth {{[0-9]+}}, 126(1)
+; CHECK-DAG: stb {{[0-9]+}}, 135(1)
+; CHECK-DAG: sth {{[0-9]+}}, 133(1)
+; CHECK-DAG: stw {{[0-9]+}}, 140(1)
+; CHECK-DAG: stb {{[0-9]+}}, 151(1)
+; CHECK-DAG: stw {{[0-9]+}}, 147(1)
+; CHECK-DAG: sth {{[0-9]+}}, 158(1)
+; CHECK-DAG: stw {{[0-9]+}}, 154(1)
+; CHECK-DAG: stb {{[0-9]+}}, 167(1)
+; CHECK-DAG: sth {{[0-9]+}}, 165(1)
+; CHECK-DAG: stw {{[0-9]+}}, 161(1)
 }
 
 define internal i32 @callee2(i32 %z1, i32 %z2, i32 %z3, i32 %z4, i32 %z5, i32 %z6, i32 %z7, i32 %z8, %struct.t1* byval %v1, %struct.t2* byval %v2, %struct.t3* byval %v3, %struct.t4* byval %v4, %struct.t5* byval %v5, %struct.t6* byval %v6, %struct.t7* byval %v7) nounwind {
 entry:
   %z1.addr = alloca i32, align 4
   %z2.addr = alloca i32, align 4
   %z3.addr = alloca i32, align 4
   %z4.addr = alloca i32, align 4

test/CodeGen/PowerPC/structsinregs.ll

entry:
   %5 = bitcast %struct.s6* %p6 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %5, i8* bitcast ({ i32, i16, [2 x i8] }* @caller1.p6 to i8*), i64 8, i32 4, i1 false)
   %6 = bitcast %struct.s7* %p7 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %6, i8* bitcast ({ i32, i16, i8, i8 }* @caller1.p7 to i8*), i64 8, i32 4, i1 false)
   %call = call i32 @callee1(%struct.s1* byval %p1, %struct.s2* byval %p2, %struct.s3* byval %p3, %struct.s4* byval %p4, %struct.s5* byval %p5, %struct.s6* byval %p6, %struct.s7* byval %p7)
   ret i32 %call
 
 ; CHECK-LABEL: caller1
-; CHECK: ld 9, 112(31)
-; CHECK: ld 8, 120(31)
-; CHECK: ld 7, 128(31)
-; CHECK: lwz 6, 136(31)
-; CHECK: lwz 5, 144(31)
-; CHECK: lhz 4, 152(31)
-; CHECK: lbz 3, 160(31)
+; CHECK-DAG: ld 9, 112(31)
+; CHECK-DAG: ld 8, 120(31)
+; CHECK-DAG: ld 7, 128(31)
+; CHECK-DAG: lwz 6, 136(31)
+; CHECK-DAG: lwz 5, 144(31)
+; CHECK-DAG: lhz 4, 152(31)
+; CHECK-DAG: lbz 3, 160(31)
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 
 define internal i32 @callee1(%struct.s1* byval %v1, %struct.s2* byval %v2, %struct.s3* byval %v3, %struct.s4* byval %v4, %struct.s5* byval %v5, %struct.s6* byval %v6, %struct.s7* byval %v7) nounwind {
 entry:
   %a = getelementptr inbounds %struct.s1, %struct.s1* %v1, i32 0, i32 0
   %0 = load i8, i8* %a, align 1
entry:
   %5 = bitcast %struct.t6* %p6 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %5, i8* bitcast (%struct.t6* @caller2.p6 to i8*), i64 6, i32 1, i1 false)
   %6 = bitcast %struct.t7* %p7 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %6, i8* bitcast (%struct.t7* @caller2.p7 to i8*), i64 7, i32 1, i1 false)
   %call = call i32 @callee2(%struct.t1* byval %p1, %struct.t2* byval %p2, %struct.t3* byval %p3, %struct.t4* byval %p4, %struct.t5* byval %p5, %struct.t6* byval %p6, %struct.t7* byval %p7)
   ret i32 %call
 
 ; CHECK-LABEL: caller2
-; CHECK: stb {{[0-9]+}}, 71(1)
-; CHECK: sth {{[0-9]+}}, 69(1)
-; CHECK: stb {{[0-9]+}}, 87(1)
-; CHECK: stw {{[0-9]+}}, 83(1)
-; CHECK: sth {{[0-9]+}}, 94(1)
-; CHECK: stw {{[0-9]+}}, 90(1)
-; CHECK: stb {{[0-9]+}}, 103(1)
-; CHECK: sth {{[0-9]+}}, 101(1)
-; CHECK: stw {{[0-9]+}}, 97(1)
-; CHECK: ld 9, 96(1)
-; CHECK: ld 8, 88(1)
-; CHECK: ld 7, 80(1)
-; CHECK: lwz 6, 136(31)
-; CHECK: ld 5, 64(1)
-; CHECK: lhz 4, 152(31)
-; CHECK: lbz 3, 160(31)
+; CHECK-DAG: stb {{[0-9]+}}, 71(1)
+; CHECK-DAG: sth {{[0-9]+}}, 69(1)
+; CHECK-DAG: stb {{[0-9]+}}, 87(1)
+; CHECK-DAG: stw {{[0-9]+}}, 83(1)
+; CHECK-DAG: sth {{[0-9]+}}, 94(1)
+; CHECK-DAG: stw {{[0-9]+}}, 90(1)
+; CHECK-DAG: stb {{[0-9]+}}, 103(1)
+; CHECK-DAG: sth {{[0-9]+}}, 101(1)
+; CHECK-DAG: stw {{[0-9]+}}, 97(1)
+; CHECK-DAG: ld 9, 96(1)
+; CHECK-DAG: ld 8, 88(1)
+; CHECK-DAG: ld 7, 80(1)
+; CHECK-DAG: lwz 6, 136(31)
+; CHECK-DAG: ld 5, 64(1)
+; CHECK-DAG: lhz 4, 152(31)
+; CHECK-DAG: lbz 3, 160(31)
 }
 
 define internal i32 @callee2(%struct.t1* byval %v1, %struct.t2* byval %v2, %struct.t3* byval %v3, %struct.t4* byval %v4, %struct.t5* byval %v5, %struct.t6* byval %v6, %struct.t7* byval %v7) nounwind {
 entry:
   %a = getelementptr inbounds %struct.t1, %struct.t1* %v1, i32 0, i32 0
   %0 = load i8, i8* %a, align 1
   %conv = zext i8 %0 to i32
   %a1 = getelementptr inbounds %struct.t2, %struct.t2* %v2, i32 0, i32 0

test/CodeGen/X86/add.ll

 
 define void @test12(i64* inreg %a) nounwind {
   %aa = load i64, i64* %a
   %b = add i64 %aa, 2147483648
   store i64 %b, i64* %a
   ret void
 ; X32-LABEL: test12:
 ; X32: addl (%
-; X32-NEXT: adcl $0,
+; X32: adcl $0,
 ; X64-LABEL: test12:
 ; X64: subq $-2147483648, (%
 }
 
 define void @test13(i64* inreg %a) nounwind {
   %aa = load i64, i64* %a
   %b = add i64 %aa, 128
   store i64 %b, i64* %a
   ret void
 
 ; X32-LABEL: test13:
 ; X32: addl (%
-; X32-NEXT: adcl $0,
+; X32: adcl $0,
 ; X64-LABEL: test13:
 ; X64: subq $-128, (%
 }

test/CodeGen/X86/musttail.ll

 declare void @t3_callee(i32)
 
 ; Test that we actually copy in and out stack arguments that aren't forwarded
 ; without modification.
 define i32 @t4({}* %fn, i32 %n, i32 %r) {
 ; CHECK-LABEL: t4:
 ; CHECK: incl %[[r:.*]]
 ; CHECK: decl %[[n:.*]]
-; CHECK: movl %[[r]], {{[0-9]+}}(%esp)
-; CHECK: movl %[[n]], {{[0-9]+}}(%esp)
+; CHECK-DAG: movl %[[r]], {{[0-9]+}}(%esp)
+; CHECK-DAG: movl %[[n]], {{[0-9]+}}(%esp)
 ; CHECK: jmpl *%{{.*}}
 
 entry:
   %r1 = add i32 %r, 1
   %n1 = sub i32 %n, 1
   %fn_cast = bitcast {}* %fn to i32 ({}*, i32, i32)*
   %r2 = musttail call i32 %fn_cast({}* %fn, i32 %n1, i32 %r1)
   ret i32 %r2
 }
 
 ; Combine the complex stack frame with the parameter modification.
 define i32 @t5({}* %fn, i32 %n, i32 %r) alignstack(32) {
 ; CHECK-LABEL: t5:
 ; CHECK: pushl %ebp
 ; CHECK: movl %esp, %ebp
 ; CHECK: pushl %esi
 ;   Align the stack.
 ; CHECK: andl $-32, %esp
 ; CHECK: movl %esp, %esi
 ;   Modify the args.
 ; CHECK: incl %[[r:.*]]
 ; CHECK: decl %[[n:.*]]
 ;   Store them through ebp, since that's the only stable arg pointer.
-; CHECK: movl %[[r]], {{[0-9]+}}(%ebp)
-; CHECK: movl %[[n]], {{[0-9]+}}(%ebp)
+; CHECK-DAG: movl %[[r]], {{[0-9]+}}(%ebp)
+; CHECK-DAG: movl %[[n]], {{[0-9]+}}(%ebp)
 ;   Epilogue.
 ; CHECK: leal {{[-0-9]+}}(%ebp), %esp
 ; CHECK: popl %esi
 ; CHECK: popl %ebp
 ; CHECK: jmpl *%{{.*}}
 
 entry:
   %a = alloca i8, i32 %n
   call void @capture(i8* %a)
   %r1 = add i32 %r, 1
   %n1 = sub i32 %n, 1
   %fn_cast = bitcast {}* %fn to i32 ({}*, i32, i32)*
   %r2 = musttail call i32 %fn_cast({}* %fn, i32 %n1, i32 %r1)
   ret i32 %r2
 }

test/CodeGen/X86/rotate.ll

; 64-NEXT:    retq
   %D = or i8 %B, %C
   ret i8 %D
 }
 
 define void @rotr1_64_mem(i64* %Aptr) nounwind {
 ; 32-LABEL: rotr1_64_mem:
 ; 32:       # BB#0:
 ; 32-NEXT:    pushl %esi
-; 32-NEXT:    movl 8(%esp), %eax
+; 32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; 32-NEXT:    movl (%eax), %ecx
 ; 32-NEXT:    movl 4(%eax), %edx
 ; 32-NEXT:    movl %edx, %esi
 ; 32-NEXT:    shldl $31, %ecx, %esi
 ; 32-NEXT:    shldl $31, %edx, %ecx
 ; 32-NEXT:    movl %ecx, 4(%eax)
 ; 32-NEXT:    movl %esi, (%eax)
 ; 32-NEXT:    popl %esi
-
+; 32-NEXT:    retl
+;
 ; 64-LABEL: rotr1_64_mem:
 ; 64:       # BB#0:
 ; 64-NEXT:    rorq (%rdi)
 ; 64-NEXT:    retq
+
   %A = load i64, i64 *%Aptr
   %B = shl i64 %A, 63
   %C = lshr i64 %A, 1
   %D = or i64 %B, %C
   store i64 %D, i64* %Aptr
   ret void
 }
 
 define void @rotr1_32_mem(i32* %Aptr) nounwind {
 ; 32-LABEL: rotr1_32_mem:
 ; 32:       # BB#0:
-; 32-NEXT:    movl 4(%esp), %eax
+; 32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; 32-NEXT:    rorl (%eax)
 ; 32-NEXT:    retl
 ;
 ; 64-LABEL: rotr1_32_mem:
 ; 64:       # BB#0:
 ; 64-NEXT:    rorl (%rdi)
 ; 64-NEXT:    retq
   %A = load i32, i32 *%Aptr
   %B = shl i32 %A, 31
   %C = lshr i32 %A, 1
   %D = or i32 %B, %C
   store i32 %D, i32* %Aptr
   ret void
 }
 
 define void @rotr1_16_mem(i16* %Aptr) nounwind {
 ; 32-LABEL: rotr1_16_mem:
 ; 32:       # BB#0:
-; 32-NEXT:    movl 4(%esp), %eax
+; 32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; 32-NEXT:    rorw (%eax)
 ; 32-NEXT:    retl
 ;
 ; 64-LABEL: rotr1_16_mem:
 ; 64:       # BB#0:
 ; 64-NEXT:    rorw (%rdi)
 ; 64-NEXT:    retq
   %A = load i16, i16 *%Aptr
   %B = shl i16 %A, 15
   %C = lshr i16 %A, 1
   %D = or i16 %B, %C
   store i16 %D, i16* %Aptr
   ret void
 }
 
 define void @rotr1_8_mem(i8* %Aptr) nounwind {
 ; 32-LABEL: rotr1_8_mem:
 ; 32:       # BB#0:
-; 32-NEXT:    movl 4(%esp), %eax
+; 32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; 32-NEXT:    rorb (%eax)
 ; 32-NEXT:    retl
 ;
 ; 64-LABEL: rotr1_8_mem:
 ; 64:       # BB#0:
 ; 64-NEXT:    rorb (%rdi)
 ; 64-NEXT:    retq
   %A = load i8, i8 *%Aptr
   %B = shl i8 %A, 7
   %C = lshr i8 %A, 1
   %D = or i8 %B, %C
   store i8 %D, i8* %Aptr
   ret void
 }