[ARM] Cortex-M4 schedule additions

[dmgreen]

Summary

This is an attempt to fill in some of the missing instructions from the Cortex-M4 schedule, and make it easier to do the same for other ARM cpus.

• Some instructions are marked as hasNoSchedulingInfo as they are pseudos or otherwise do not require scheduling info
• A lot of features have been marked not supported
• Some WriteRes's have been added for cvt instructions.
• Some extra instruction latencies have been added, notably by relaxing the regex for dsp instruction to catch more cases, and some fp instructions.

This goes a long way to get the CompleteModel working for this CPU. It does not go as far as to get all scheduling info for all output operands correct.

[SjoerdMeijer]

Looks all like very mechanical changes to me. The only thing I am not a big fan of is this UnsupportedFeatures, but I understood that there is not really a much better alternative.

The only question I have is related to the Predicates=[IsWindows] addition, which is a functional change. Do we need a test for that?

[dmgreen]

Thanks for taking a look.

I believe that those instructions would only be generated on windows targets from ISelLowering, from functions like LowerDYNAMIC_STACKALLOC. As in, they require IsWindows anyway by the fact that they will not be generated anywhere else, we are just teaching ISel the same thing!

But looking again I think it's simpler to make that hasNoSchedulingInfo as it usesCustomInserter anyway, and the scheduler will not see it in the normal scheduler. They get expanded post-isel into a BL call to __chkstk and a sub.

[dmgreen]

Change WIN__CHKSTK from IsWindows to hasNoSchedulingInfo

[SjoerdMeijer]

Cheers, nice one, LGTM

llvm/trunk/lib/Target/ARM/ARMInstrFormats.td

class AsmPseudoInst<string asm, dag iops, dag oops = (outs)>
   : InstTemplate<AddrModeNone, 0, IndexModeNone, Pseudo, GenericDomain,
                  "", NoItinerary> {
   let OutOperandList = oops;
   let InOperandList = iops;
   let Pattern = [];
   let isCodeGenOnly = 0; // So we get asm matcher for it.
   let AsmString = asm;
   let isPseudo = 1;
+  let hasNoSchedulingInfo = 1;
 }
 
 class ARMAsmPseudo<string asm, dag iops, dag oops = (outs)>
   : AsmPseudoInst<asm, iops, oops>, Requires<[IsARM]>;
 class tAsmPseudo<string asm, dag iops, dag oops = (outs)>
   : AsmPseudoInst<asm, iops, oops>, Requires<[IsThumb]>;
 class t2AsmPseudo<string asm, dag iops, dag oops = (outs)>
   : AsmPseudoInst<asm, iops, oops>, Requires<[IsThumb2]>;

llvm/trunk/lib/Target/ARM/ARMInstrInfo.td

 //===----------------------------------------------------------------------===//
 //  Miscellaneous Instructions.
 //
 
 /// CONSTPOOL_ENTRY - This instruction represents a floating constant pool in
 /// the function.  The first operand is the ID# for this instruction, the second
 /// is the index into the MachineConstantPool that this is, the third is the
 /// size in bytes of this constant pool entry.
-let hasSideEffects = 0, isNotDuplicable = 1 in
+let hasSideEffects = 0, isNotDuplicable = 1, hasNoSchedulingInfo = 1 in
 def CONSTPOOL_ENTRY :
 PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
                     i32imm:$size), NoItinerary, []>;
 
 /// A jumptable consisting of direct 32-bit addresses of the destination basic
 /// blocks (either absolute, or relative to the start of the jump-table in PIC
 /// mode). Used mostly in ARM and Thumb-1 modes.
 def JUMPTABLE_ADDRS :

 // Armv8.5-A speculation barrier
 def SB : AInoP<(outs), (ins), MiscFrm, NoItinerary, "sb", "", []>,
          Requires<[IsARM, HasSB]>, Sched<[]> {
   let Inst{31-0} = 0xf57ff070;
   let Unpredictable = 0x000fff0f;
   let hasSideEffects = 1;
 }
 
-let usesCustomInserter = 1, Defs = [CPSR] in {
-
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in {
   // Pseudo instruction that combines movs + predicated rsbmi
   // to implement integer ABS
   def ABS : ARMPseudoInst<(outs GPR:$dst), (ins GPR:$src), 8, NoItinerary, []>;
 }
 
-let usesCustomInserter = 1, Defs = [CPSR] in {
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in {
     def COPY_STRUCT_BYVAL_I32 : PseudoInst<
       (outs), (ins GPR:$dst, GPR:$src, i32imm:$size, i32imm:$alignment),
       NoItinerary,
       [(ARMcopystructbyval GPR:$dst, GPR:$src, imm:$size, imm:$alignment)]>;
 }
 
 let hasPostISelHook = 1, Constraints = "$newdst = $dst, $newsrc = $src" in {
     // %newsrc, %newdst = MEMCPY %dst, %src, N, ...N scratch regs...

 // 4k bytes in one go. Touching the stack at 4K increments is necessary to
 // ensure that the guard pages used by the OS virtual memory manager are
 // allocated in correct sequence.
 // The main point of having separate instruction are extra unmodelled effects
 // (compared to ordinary calls) like stack pointer change.
 
 def win__chkstk : SDNode<"ARMISD::WIN__CHKSTK", SDTNone,
                       [SDNPHasChain, SDNPSideEffect]>;
-let usesCustomInserter = 1, Uses = [R4], Defs = [R4, SP] in
+let usesCustomInserter = 1, Uses = [R4], Defs = [R4, SP], hasNoSchedulingInfo = 1 in
   def WIN__CHKSTK : PseudoInst<(outs), (ins), NoItinerary, [(win__chkstk)]>;
 
 def win__dbzchk : SDNode<"ARMISD::WIN__DBZCHK", SDT_WIN__DBZCHK,
                          [SDNPHasChain, SDNPSideEffect, SDNPOutGlue]>;
-let usesCustomInserter = 1, Defs = [CPSR] in
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in
   def WIN__DBZCHK : PseudoInst<(outs), (ins tGPR:$divisor), NoItinerary,
                                [(win__dbzchk tGPR:$divisor)]>;
 
 //===----------------------------------------------------------------------===//
 // TLS Instructions
 //
 
 // __aeabi_read_tp preserves the registers r1-r3.
def : InstAlias<"umull${s}${p} $RdLo, $RdHi, $Rn, $Rm",
             (UMULL GPR:$RdLo, GPR:$RdHi, GPR:$Rn, GPR:$Rm, pred:$p, cc_out:$s), 0>,
          Requires<[IsARM, NoV6]>;
 
 // 'it' blocks in ARM mode just validate the predicates. The IT itself
 // is discarded.
 def ITasm : ARMAsmPseudo<"it$mask $cc", (ins it_pred:$cc, it_mask:$mask)>,
          ComplexDeprecationPredicate<"IT">;
 
-let mayLoad = 1, mayStore =1, hasSideEffects = 1 in
+let mayLoad = 1, mayStore =1, hasSideEffects = 1, hasNoSchedulingInfo = 1 in
 def SPACE : PseudoInst<(outs GPR:$Rd), (ins i32imm:$size, GPR:$Rn),
                        NoItinerary,
                        [(set GPR:$Rd, (int_arm_space timm:$size, GPR:$Rn))]>;
 
 //===----------------------------------
 // Atomic cmpxchg for -O0
 //===----------------------------------
 
def CMP_SWAP_64 : PseudoInst<(outs GPRPair:$Rd, GPR:$temp),
                              NoItinerary, []>, Sched<[]>;
 }
 
 def CompilerBarrier : PseudoInst<(outs), (ins i32imm:$ordering), NoItinerary,
                                  [(atomic_fence imm:$ordering, 0)]> {
   let hasSideEffects = 1;
   let Size = 0;
   let AsmString = "@ COMPILER BARRIER";
+  let hasNoSchedulingInfo = 1;
 }

llvm/trunk/lib/Target/ARM/ARMInstrThumb2.td

 def t2LDRpci_pic : PseudoInst<(outs rGPR:$dst), (ins i32imm:$addr, pclabel:$cp),
                    IIC_iLoadiALU,
               [(set rGPR:$dst, (ARMpic_add (load (ARMWrapper tconstpool:$addr)),
                                            imm:$cp))]>,
                Requires<[IsThumb2]>;
 
 // Pseudo isntruction that combines movs + predicated rsbmi
 // to implement integer ABS
-let usesCustomInserter = 1, Defs = [CPSR] in {
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in {
 def t2ABS : PseudoInst<(outs rGPR:$dst), (ins rGPR:$src),
                        NoItinerary, []>, Requires<[IsThumb2]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Coprocessor load/store -- for disassembly only
 //
 class T2CI<bits<4> op31_28, dag oops, dag iops, string opc, string asm, list<dag> pattern>
class t2BF<dag iops, string asm, string ops>
 
   let Predicates = [IsThumb2, HasV8_1MMainline, HasLOB];
 }
 
 def t2BF_LabelPseudo
   : t2PseudoInst<(outs ), (ins pclabel:$cp), 0, NoItinerary, []> {
   let isTerminator = 1;
   let Predicates = [IsThumb2, HasV8_1MMainline, HasLOB];
+  let hasNoSchedulingInfo = 1;
 }
 
 def t2BFi : t2BF<(ins bflabel_u4:$b_label, bflabel_s16:$label, pred:$p),
                  !strconcat("bf", "${p}"), "$b_label, $label"> {
   bits<4> b_label;
   bits<16> label;
 
   let Inst{26-23} = b_label{3-0};

llvm/trunk/lib/Target/ARM/ARMInstrVFP.td

defm : VFPDTAnyInstAlias<"vpop${p}", "$r",
                          (VLDMDIA_UPD SP, pred:$p, dpr_reglist:$r)>;
 
 // FLDMX, FSTMX - Load and store multiple unknown precision registers for
 // pre-armv6 cores.
 // These instruction are deprecated so we don't want them to get selected.
 // However, there is no UAL syntax for them, so we keep them around for
 // (dis)assembly only.
 multiclass vfp_ldstx_mult<string asm, bit L_bit> {
-  let Predicates = [HasFPRegs] in {
+  let Predicates = [HasFPRegs], hasNoSchedulingInfo = 1 in {
   // Unknown precision
   def XIA :
     AXXI4<(outs), (ins GPR:$Rn, pred:$p, dpr_reglist:$regs, variable_ops),
           IndexModeNone, !strconcat(asm, "iax${p}\t$Rn, $regs"), "", []> {
     let Inst{24-23} = 0b01;       // Increment After
     let Inst{21}    = 0;          // No writeback
     let Inst{20}    = L_bit;
   }
def VTOSHS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1110, 0b1010, 0,
              Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
 }
 
 def VTOUHS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1111, 0b1010, 0,
                        (outs SPR:$dst), (ins SPR:$a, fbits16:$fbits),
-                 IIC_fpCVTSI, "vcvt", ".u16.f32\t$dst, $a, $fbits", []> {
+                 IIC_fpCVTSI, "vcvt", ".u16.f32\t$dst, $a, $fbits", []>,
+             Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
 }
 
 def VTOSLS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1110, 0b1010, 1,
                        (outs SPR:$dst), (ins SPR:$a, fbits32:$fbits),
-                 IIC_fpCVTSI, "vcvt", ".s32.f32\t$dst, $a, $fbits", []> {
+                 IIC_fpCVTSI, "vcvt", ".s32.f32\t$dst, $a, $fbits", []>,
+             Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
 }
 
 def VTOULS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1111, 0b1010, 1,
                        (outs SPR:$dst), (ins SPR:$a, fbits32:$fbits),
-                 IIC_fpCVTSI, "vcvt", ".u32.f32\t$dst, $a, $fbits", []> {
+                 IIC_fpCVTSI, "vcvt", ".u32.f32\t$dst, $a, $fbits", []>,
+             Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
 }
 
 def VTOSHD : AVConv1XInsD_Encode<0b11101, 0b11, 0b1110, 0b1011, 0,
                        (outs DPR:$dst), (ins DPR:$a, fbits16:$fbits),
                  IIC_fpCVTDI, "vcvt", ".s16.f64\t$dst, $a, $fbits", []>,

llvm/trunk/lib/Target/ARM/ARMScheduleM4.td

 def CortexM4Model : SchedMachineModel {
   let IssueWidth        = 1; // Only IT can be dual-issued, so assume single-issue
   let MicroOpBufferSize = 0; // In-order
   let LoadLatency       = 2; // Latency when not pipelined, not pc-relative
   let MispredictPenalty = 2; // Best case branch taken cost
   let PostRAScheduler   = 1;
 
   let CompleteModel = 0;
+  let UnsupportedFeatures = [IsARM, HasNEON, HasDotProd, HasZCZ, HasMVEInt,
+          IsNotMClass, HasDPVFP, HasFPARMv8, HasFullFP16, Has8MSecExt, HasV8,
+          HasV8_3a, HasTrustZone, HasDFB, IsWindows];
 }
 
 
 // We model the entire cpu as a single pipeline with a BufferSize = 0 since
 // Cortex-M4 is in-order.
 
 def M4Unit : ProcResource<1> { let BufferSize = 0; }
 

 def : M4UnitL2<WriteLd>;
 def : M4UnitL2<WriteMAC32>;
 def : M4UnitL2<WriteMAC64Hi>;
 def : M4UnitL2<WriteMAC64Lo>;
 def : M4UnitL2<WriteMAC16>;
 def : M4UnitL2<WriteDIV>;
 
 def : M4UnitL2I<(instregex "(t|t2)LDM")>;
+def : M4UnitL2I<(instregex "(t|t2)LDR")>;
 
 
 // Stores we use a latency of 1 as they have no outputs
 
 def : M4UnitL1<WriteST>;
 def : M4UnitL1I<(instregex "(t|t2)STM")>;
 
 

 def : M4UnitL1<WriteMUL32>;
 def : M4UnitL1<WriteMUL64Hi>;
 def : M4UnitL1<WriteMUL64Lo>;
 def : M4UnitL1<WriteMUL16>;
 def : M4UnitL1<WriteNoop>;
 def : M4UnitL1<WritePreLd>;
 def : M4UnitL1I<(instregex "(t|t2)MOV")>;
 def : M4UnitL1I<(instrs COPY)>;
-def : M4UnitL1I<(instregex "t2IT")>;
-def : M4UnitL1I<(instregex "t2SEL", "t2USAD8",
-    "t2(S|Q|SH|U|UQ|UH)(ADD16|ASX|SAX|SUB16|ADD8|SUB8)", "t2USADA8", "(t|t2)REV")>;
+def : M4UnitL1I<(instregex "t2IT", "t2MSR", "t2MRS")>;
+def : M4UnitL1I<(instregex "t2CLREX")>;
+def : M4UnitL1I<(instregex "t2SEL", "t2USAD8", "t2SML[AS]",
+    "t2(S|Q|SH|U|UQ|UH|QD)(ADD|ASX|SAX|SUB)", "t2USADA8", "(t|t2)REV")>;
+
+// These instructions are not of much interest to scheduling as they will not
+// be generated or it is not very useful to schedule them. They are here to make
+// the model more complete.
+def : M4UnitL1I<(instregex "t2CDP", "t2LDC", "t2MCR", "t2MRC", "t2MRRC", "t2STC")>;
+def : M4UnitL1I<(instregex "tCPS", "t2ISB", "t2DSB", "t2DMB", "t2?HINT$")>;
+def : M4UnitL1I<(instregex "t2?UDF$", "tBKPT", "t2DBG")>;
+def : M4UnitL1I<(instregex "t?2?Int_eh_sjlj_", "tADDframe", "t?ADJCALL")>;
+def : M4UnitL1I<(instregex "CMP_SWAP", "JUMPTABLE", "MEMCPY")>;
+def : M4UnitL1I<(instregex "VSETLNi32", "VGETLNi32")>;
 
 def : ReadAdvance<ReadALU, 0>;
 def : ReadAdvance<ReadALUsr, 0>;
 def : ReadAdvance<ReadMUL, 0>;
 def : ReadAdvance<ReadMAC, 0>;
 
 // Most FP instructions are single-cycle latency, except MAC's, Div's and Sqrt's.
 // Loads still take 2 cycles.

 def : M4UnitL1<WriteVLD1>;
 def : M4UnitL1<WriteVLD2>;
 def : M4UnitL1<WriteVLD3>;
 def : M4UnitL1<WriteVLD4>;
 def : M4UnitL1<WriteVST1>;
 def : M4UnitL1<WriteVST2>;
 def : M4UnitL1<WriteVST3>;
 def : M4UnitL1<WriteVST4>;
+def : M4UnitL1I<(instregex "VMOVS", "FCONSTS", "VCMP", "VNEG", "VABS")>;
+def : M4UnitL2I<(instregex "VMOVD")>;
+def : M4UnitL1I<(instregex "VMRS", "VMSR", "FMSTAT")>;
 
 def : ReadAdvance<ReadFPMUL, 0>;
 def : ReadAdvance<ReadFPMAC, 0>;
 
 }

llvm/trunk/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll

 ; RUN: llc -mtriple=thumbv7em -mcpu=cortex-m4 -O3 %s -o - | FileCheck %s
 
 ; TODO: Think we should be able to use smlsdx/smlsldx here.
 
 ; CHECK-LABEL: complex_dot_prod
 
 ; CHECK: smulbb
 ; CHECK: smultt
 ; CHECK: smlalbb
 ; CHECK: smultt
-; CHECK: smlaldx
 ; CHECK: smlalbb
-; CHECK: smlaldx
 ; CHECK: smultt
 ; CHECK: smlalbb
-; CHECK: smlaldx
 ; CHECK: smultt
+; CHECK: smlaldx
+; CHECK: smlaldx
+; CHECK: smlaldx
 ; CHECK: pop.w  {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 define dso_local arm_aapcscc void @complex_dot_prod(i16* nocapture readonly %pSrcA, i16* nocapture readonly %pSrcB, i32* nocapture %realResult, i32* nocapture %imagResult) {
 entry:
   %incdec.ptr = getelementptr inbounds i16, i16* %pSrcA, i32 1
   %0 = load i16, i16* %pSrcA, align 2
   %incdec.ptr1 = getelementptr inbounds i16, i16* %pSrcA, i32 2
   %1 = load i16, i16* %incdec.ptr, align 2
   %incdec.ptr2 = getelementptr inbounds i16, i16* %pSrcB, i32 1

llvm/trunk/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll

 ; CHECK-LE-NEXT:    subs r3, #2
 ; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    .p2align 2
 ; CHECK-LE-NEXT:  .LBB0_2: @ %for.body
 ; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-LE-NEXT:    ldr lr, [r3, #2]!
 ; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
-; CHECK-LE-NEXT:    subs r0, #1
 ; CHECK-LE-NEXT:    sxtah r1, r1, lr
+; CHECK-LE-NEXT:    subs r0, #1
 ; CHECK-LE-NEXT:    smlad r12, r4, lr, r12
 ; CHECK-LE-NEXT:    bne .LBB0_2
 ; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-LE-NEXT:    add.w r0, r12, r1
 ; CHECK-LE-NEXT:    pop {r4, pc}
 ; CHECK-LE-NEXT:  .LBB0_4:
 ; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    movs r1, #0

 ; CHECK-BE-NEXT:    subs r2, #2
 ; CHECK-BE-NEXT:    subs r3, #2
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    .p2align 2
 ; CHECK-BE-NEXT:  .LBB0_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
-; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
-; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh r5, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r4, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
-; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    subs r0, #1
-; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    smlabb r5, r5, lr, r12
 ; CHECK-BE-NEXT:    add r1, lr
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    smlabb r12, r6, r4, r5
 ; CHECK-BE-NEXT:    bne .LBB0_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
 ; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-BE-NEXT:  .LBB0_4:
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    add.w r0, r12, r1

 ; CHECK-BE-NEXT:    .p2align 2
 ; CHECK-BE-NEXT:  .LBB1_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
 ; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
 ; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
 ; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    mul r1, lr, r1
 ; CHECK-BE-NEXT:    bne .LBB1_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
 ; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-BE-NEXT:  .LBB1_4:
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0

 define i32 @mul_top_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
 ; CHECK-LE-LABEL: mul_top_user:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-LE-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-LE-NEXT:    cmp r0, #1
 ; CHECK-LE-NEXT:    blt .LBB2_4
 ; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-LE-NEXT:    subs r2, #2
+; CHECK-LE-NEXT:    sub.w lr, r2, #2
 ; CHECK-LE-NEXT:    subs r3, #2
 ; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    .p2align 2
 ; CHECK-LE-NEXT:  .LBB2_2: @ %for.body
 ; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
-; CHECK-LE-NEXT:    ldr lr, [r3, #2]!
-; CHECK-LE-NEXT:    asrs r5, r4, #16
-; CHECK-LE-NEXT:    smlad r12, r4, lr, r12
+; CHECK-LE-NEXT:    ldr r2, [lr, #2]!
+; CHECK-LE-NEXT:    ldr r4, [r3, #2]!
+; CHECK-LE-NEXT:    asrs r5, r2, #16
+; CHECK-LE-NEXT:    smlad r12, r2, r4, r12
 ; CHECK-LE-NEXT:    subs r0, #1
 ; CHECK-LE-NEXT:    mul r1, r5, r1
 ; CHECK-LE-NEXT:    bne .LBB2_2
 ; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-LE-NEXT:    add.w r0, r12, r1
 ; CHECK-LE-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-LE-NEXT:  .LBB2_4:
 ; CHECK-LE-NEXT:    mov.w r12, #0

 ; CHECK-BE-NEXT:    .p2align 2
 ; CHECK-BE-NEXT:  .LBB2_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
 ; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
 ; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
 ; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    mul r1, r6, r1
 ; CHECK-BE-NEXT:    bne .LBB2_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
 ; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-BE-NEXT:  .LBB2_4:
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0

 ; CHECK-LE-NEXT:  .LBB3_4:
 ; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    add.w r0, r12, r1
 ; CHECK-LE-NEXT:    pop {r4, r5, r7, pc}
 ;
 ; CHECK-BE-LABEL: and_user:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-BE-NEXT:    push {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, r7, lr}
 ; CHECK-BE-NEXT:    cmp r0, #1
 ; CHECK-BE-NEXT:    blt .LBB3_4
 ; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-BE-NEXT:    subs r2, #2
 ; CHECK-BE-NEXT:    subs r3, #2
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    .p2align 2
 ; CHECK-BE-NEXT:  .LBB3_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
-; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
-; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
-; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
-; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    uxth.w lr, lr
-; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    ldrsh r5, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r4, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r7, [r2, #2]
+; CHECK-BE-NEXT:    uxth.w r6, lr
+; CHECK-BE-NEXT:    smlabb r5, r5, lr, r12
+; CHECK-BE-NEXT:    smlabb r12, r7, r4, r5
 ; CHECK-BE-NEXT:    subs r0, #1
-; CHECK-BE-NEXT:    mul r1, lr, r1
+; CHECK-BE-NEXT:    mul r1, r6, r1
 ; CHECK-BE-NEXT:    bne .LBB3_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
-; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:    pop {r4, r5, r6, r7, pc}
 ; CHECK-BE-NEXT:  .LBB3_4:
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
-; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:    pop {r4, r5, r6, r7, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
 
 for.body.preheader:
   %.pre = load i16, i16* %arg3, align 2
   %.pre27 = load i16, i16* %arg2, align 2
   br label %for.body

 ; CHECK-BE-NEXT:    .p2align 2
 ; CHECK-BE-NEXT:  .LBB4_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh r1, [r3, #2]!
 ; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
 ; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
 ; CHECK-BE-NEXT:    smlabb r4, r4, r1, r12
-; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
 ; CHECK-BE-NEXT:    eor.w r6, r1, lr
-; CHECK-BE-NEXT:    mul r1, r6, r1
+; CHECK-BE-NEXT:    muls r1, r6, r1
+; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    lsl.w lr, r1, #16
 ; CHECK-BE-NEXT:    bne .LBB4_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, lr
 ; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-BE-NEXT:  .LBB4_4:
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    mov.w lr, #0

llvm/trunk/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll

 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
-; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: bne .LBB0_1
 
 for.body:
   %A3 = phi i32 [ %add9.us.i.3361.i, %for.body ], [ 0, %entry ]
   %j.026.us.i.i = phi i32 [ %inc.us.i.3362.i, %for.body ], [ 0, %entry ]
   %A4 = phi i32 [ %add9.us.i.1.3.i, %for.body ], [ 0, %entry ]
   %A5 = phi i32 [ %add9.us.i.2.3.i, %for.body ], [ 0, %entry ]
   %A6 = phi i32 [ %add9.us.i.3.3.i, %for.body ], [ 0, %entry ]