[AAch64] Optimize muls with operands having enough zero bits.

[bipmis]

Summary

Muls with 64bit operands where each of the operand is having top 32 bits as zero, we can generate a single umull instruction on a 32bit operand.

[david-arm]

llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll

908

Perhaps it's worth pre-committing these tests so that we can see what's changed? It's not immediately obvious from the patch what effect the changes have that's all.

[dmgreen]

Can you add a couple of testcases for the edgecases too. Something like %and = and i64 %ext64, 8589934591 and %and = and i64 %ext64, 2147483647

[bipmis]

Corner case tests added. Rebase with tests committed.

[dmgreen]

LGTM Thanks.

[dmgreen]

It looks like this might be running into bootstrap issues, according to the bisect I just did.

[mstorsjo]

I also bisected a regression to this commit.

[mstorsjo]

The misoptimization occurs in this preprocessed file: https://martin.st/temp/lagarith-preproc.c

When built with clang -target aarch64-linux-gnu -S lagarith-preproc.c -o out.s -O3, the difference before/after this change looks like this:

--- good.s      2022-12-21 11:42:08.816749585 +0200
+++ bad.s       2022-12-21 11:43:20.672946992 +0200
@@ -1647,7 +1647,7 @@
        lsr     w14, w20, #1
        mov     x12, xzr
        mov     w9, wzr
-       msub    x11, x13, x20, x11
+       umsubl  x11, w13, w20, x11
        lsl     x11, x11, x10
        lsl     x10, x13, x10
        add     x11, x11, x14
@@ -1661,9 +1661,9 @@
 .LBB2_125:                              // %for.body66.i
                                         // =>This Inner Loop Header: Depth=1
        ldr     w15, [x13, x12]
-       mul     x16, x10, x15
-       mul     x15, x11, x15
-       add     x15, x15, x16, lsr #32
+       umull   x16, w10, w15
+       lsr     x17, x16, #32
+       umaddl  x15, w11, w15, x17
        lsr     x17, x15, #21
        orr     w17, w17, #0x1
        clz     w17, w17
@@ -1687,9 +1687,9 @@
 .LBB2_128:                              // %for.body89.i
                                         // =>This Inner Loop Header: Depth=1
        ldr     w15, [x13, x12]
-       mul     x16, x10, x15
-       mul     x15, x11, x15
-       add     x15, x15, x16, lsr #32
+       umull   x16, w10, w15
+       lsr     x17, x16, #32
+       umaddl  x15, w11, w15, x17
        lsr     x17, x15, #21
        orr     w17, w17, #0x1
        clz     w17, w17

The full runtime case can be reproduced on Linux on aarch64 like this:

$ git clone git://source.ffmpeg.org/ffmpeg
$ mkdir ffmpeg-build
$ cd ffmpeg-build
$ ../ffmpeg/configure --samples=$(pwd)/../samples --cc=clang
$ make fate-rsync # download test inputs
$ make -j$(nproc) fate-lagarith

The object file with the misoptimization is libavcodec/lagarith.o here.

[bipmis]

In D139411#4010230, @mstorsjo wrote:

The misoptimization occurs in this preprocessed file: https://martin.st/temp/lagarith-preproc.c

Thanks for reporting. Should be fixed with the latest commit. The bootstrap passed fine as well.

[mstorsjo]

In D139411#4010352, @bipmis wrote:

In D139411#4010230, @mstorsjo wrote:

The misoptimization occurs in this preprocessed file: https://martin.st/temp/lagarith-preproc.c

Thanks for reporting. Should be fixed with the latest commit. The bootstrap passed fine as well.

Thanks! Yes, the issue seems to be fixed now.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

def : Pat<(i64 (ineg (smullwithsignbits GPR64:$Rn, GPR64:$Rm))),
           (SMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), (EXTRACT_SUBREG $Rm, sub_32), XZR)>;
 def : Pat<(i64 (ineg (smullwithsignbits GPR64:$Rn, (sext GPR32:$Rm)))),
           (SMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), $Rm, XZR)>;
 
 def : Pat<(i64 (sub GPR64:$Ra, (smullwithsignbits GPR64:$Rn, GPR64:$Rm))),
           (SMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), (EXTRACT_SUBREG $Rm, sub_32), GPR64:$Ra)>;
 def : Pat<(i64 (sub GPR64:$Ra, (smullwithsignbits GPR64:$Rn, (sext GPR32:$Rm)))),
           (SMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), $Rm, GPR64:$Ra)>;
+
+def : Pat<(i64 (mul top32Zero:$Rn, top32Zero:$Rm)),
+          (UMADDLrrr (EXTRACT_SUBREG $Rn, sub_32), (EXTRACT_SUBREG $Rm, sub_32), XZR)>;
+def : Pat<(i64 (mul top32Zero:$Rn, (zext GPR32:$Rm))),
+          (UMADDLrrr (EXTRACT_SUBREG $Rn, sub_32), $Rm, XZR)>;
+
+def : Pat<(i64 (add (mul top32Zero:$Rn, top32Zero:$Rm), GPR64:$Ra)),
+          (UMADDLrrr (EXTRACT_SUBREG $Rn, sub_32), (EXTRACT_SUBREG $Rm, sub_32), GPR64:$Ra)>;
+def : Pat<(i64 (add (mul top32Zero:$Rn, (zext GPR32:$Rm)), GPR64:$Ra)),
+          (UMADDLrrr (EXTRACT_SUBREG $Rn, sub_32), $Rm, GPR64:$Ra)>;
+
+def : Pat<(i64 (ineg (mul top32Zero:$Rn, top32Zero:$Rm))),
+          (UMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), (EXTRACT_SUBREG $Rm, sub_32), XZR)>;
+def : Pat<(i64 (ineg (mul top32Zero:$Rn, (zext GPR32:$Rm)))),
+          (UMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), $Rm, XZR)>;
+
+def : Pat<(i64 (sub GPR64:$Ra, (mul top32Zero:$Rn, top32Zero:$Rm))),
+          (UMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), (EXTRACT_SUBREG $Rm, sub_32), GPR64:$Ra)>;
+def : Pat<(i64 (sub GPR64:$Ra, (mul GPR64:$Rn, (zext GPR32:$Rm)))),
+          (UMSUBLrrr (EXTRACT_SUBREG $Rn, sub_32), $Rm, GPR64:$Ra)>;
 } // AddedComplexity = 5
 
 def : MulAccumWAlias<"mul", MADDWrrr>;
 def : MulAccumXAlias<"mul", MADDXrrr>;
 def : MulAccumWAlias<"mneg", MSUBWrrr>;
 def : MulAccumXAlias<"mneg", MSUBXrrr>;
 def : WideMulAccumAlias<"smull", SMADDLrrr>;
 def : WideMulAccumAlias<"smnegl", SMSUBLrrr>;

llvm/test/CodeGen/AArch64/aarch64-mull-masks.ll

 ; CHECK-NEXT:    ret
 entry:
   %tmp1 = ashr i64 %a, 32
   %c = ashr i64 %b, 32
   %tmp3 = mul i64 %tmp1, %c
   ret i64 %tmp3
 }
 
 define i64 @umull_ldrb_h(ptr %x0, i16 %x1) {
 ; CHECK-LABEL: umull_ldrb_h:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smull x0, w8, w9
+; CHECK-NEXT:    umull x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext, %zext4
   ret i64 %mul
 }
 
 define i64 @umull_ldrb_h_commuted(ptr %x0, i16 %x1) {
 ; CHECK-LABEL: umull_ldrb_h_commuted:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smull x0, w9, w8
+; CHECK-NEXT:    umull x0, w9, w8
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext4, %zext
   ret i64 %mul
 }
 
 define i64 @umull_ldrh_w(ptr %x0, i32 %x1) {
 ; CHECK-LABEL: umull_ldrh_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    mul x0, x8, x9
+; CHECK-NEXT:    umull x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %zext, %zext4
   ret i64 %mul
 }
 
 define i64 @umull_ldr_b(ptr %x0, i8 %x1) {
 ; CHECK-LABEL: umull_ldr_b:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    mul x0, x8, x9
+; CHECK-NEXT:    umull x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
   %zext4 = zext i8 %x1 to i64
   %mul = mul i64 %zext, %zext4
   ret i64 %mul
 }
 
 define i64 @umull_ldr2_w(ptr %x0, i32 %x1) {
 ; CHECK-LABEL: umull_ldr2_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    mul x0, x8, x9
+; CHECK-NEXT:    umull x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %and, %zext4
   ret i64 %mul
 }
 
 define i64 @umull_ldr2_ldr2(ptr %x0, ptr %x1) {
 ; CHECK-LABEL: umull_ldr2_ldr2:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    mul x0, x8, x9
+; CHECK-NEXT:    umull x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %ext64_2 = load i64, ptr %x1
   %and2 = and i64 %ext64_2, 4294967295
   %mul = mul i64 %and, %and2
   ret i64 %mul
 }
 
 define i64 @umull_ldr2_d(ptr %x0, i64 %x1) {
 ; CHECK-LABEL: umull_ldr2_d:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    and x9, x1, #0xffffffff
-; CHECK-NEXT:    mul x0, x8, x9
+; CHECK-NEXT:    umull x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %and2 = and i64 %x1, 4294967295
   %mul = mul i64 %and, %and2
   ret i64 %mul
 }
 
 define i64 @umaddl_ldrb_h(ptr %x0, i16 %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldrb_h:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smaddl x0, w8, w9, x2
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umaddl_ldrb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldrb_h_commuted:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smaddl x0, w9, w8, x2
+; CHECK-NEXT:    umaddl x0, w9, w8, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext4, %zext
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umaddl_ldrh_w(ptr %x0, i32 %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldrh_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    madd x0, x8, x9, x2
+; CHECK-NEXT:    umaddl x0, w8, w1, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umaddl_ldr_b(ptr %x0, i8 %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldr_b:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    madd x0, x8, x9, x2
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
   %zext4 = zext i8 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umaddl_ldr2_w(ptr %x0, i32 %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldr2_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    madd x0, x8, x9, x2
+; CHECK-NEXT:    umaddl x0, w8, w1, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %and, %zext4
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umaddl_ldr2_ldr2(ptr %x0, ptr %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldr2_ldr2:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    madd x0, x8, x9, x2
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %ext64_2 = load i64, ptr %x1
   %and2 = and i64 %ext64_2, 4294967295
   %mul = mul i64 %and, %and2
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umaddl_ldr2_d(ptr %x0, i64 %x1, i64 %x2) {
 ; CHECK-LABEL: umaddl_ldr2_d:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    and x9, x1, #0xffffffff
-; CHECK-NEXT:    madd x0, x8, x9, x2
+; CHECK-NEXT:    umaddl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %and2 = and i64 %x1, 4294967295
   %mul = mul i64 %and, %and2
   %add = add i64 %mul, %x2
   ret i64 %add
 }
 
 define i64 @umnegl_ldrb_h(ptr %x0, i16 %x1) {
 ; CHECK-LABEL: umnegl_ldrb_h:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smnegl x0, w8, w9
+; CHECK-NEXT:    umnegl x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umnegl_ldrb_h_commuted(ptr %x0, i16 %x1) {
 ; CHECK-LABEL: umnegl_ldrb_h_commuted:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smnegl x0, w9, w8
+; CHECK-NEXT:    umnegl x0, w9, w8
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext4, %zext
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umnegl_ldrh_w(ptr %x0, i32 %x1) {
 ; CHECK-LABEL: umnegl_ldrh_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    mneg x0, x8, x9
+; CHECK-NEXT:    umnegl x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umnegl_ldr_b(ptr %x0, i8 %x1) {
 ; CHECK-LABEL: umnegl_ldr_b:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    mneg x0, x8, x9
+; CHECK-NEXT:    umnegl x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
   %zext4 = zext i8 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umnegl_ldr2_w(ptr %x0, i32 %x1) {
 ; CHECK-LABEL: umnegl_ldr2_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    mneg x0, x8, x9
+; CHECK-NEXT:    umnegl x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %and, %zext4
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umnegl_ldr2_ldr2(ptr %x0, ptr %x1) {
 ; CHECK-LABEL: umnegl_ldr2_ldr2:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    mneg x0, x8, x9
+; CHECK-NEXT:    umnegl x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %ext64_2 = load i64, ptr %x1
   %and2 = and i64 %ext64_2, 4294967295
   %mul = mul i64 %and, %and2
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umnegl_ldr2_d(ptr %x0, i64 %x1) {
 ; CHECK-LABEL: umnegl_ldr2_d:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    and x9, x1, #0xffffffff
-; CHECK-NEXT:    mneg x0, x8, x9
+; CHECK-NEXT:    umnegl x0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %and2 = and i64 %x1, 4294967295
   %mul = mul i64 %and, %and2
   %sub = sub i64 0, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldrb_h(ptr %x0, i16 %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldrb_h:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smsubl x0, w8, w9, x2
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldrb_h_commuted(ptr %x0, i16 %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldrb_h_commuted:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrb w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xffff
-; CHECK-NEXT:    smsubl x0, w9, w8, x2
+; CHECK-NEXT:    umsubl x0, w9, w8, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i8, ptr %x0
   %zext = zext i8 %ext64 to i64
   %zext4 = zext i16 %x1 to i64
   %mul = mul i64 %zext4, %zext
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldrh_w(ptr %x0, i32 %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldrh_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    msub x0, x8, x9, x2
+; CHECK-NEXT:    umsubl x0, w8, w1, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i16, ptr %x0
   %zext = zext i16 %ext64 to i64
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldr_b(ptr %x0, i8 %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldr_b:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    and x9, x1, #0xff
-; CHECK-NEXT:    msub x0, x8, x9, x2
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i32, ptr %x0
   %zext = zext i32 %ext64 to i64
   %zext4 = zext i8 %x1 to i64
   %mul = mul i64 %zext, %zext4
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldr2_w(ptr %x0, i32 %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldr2_w:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    mov w9, w1
-; CHECK-NEXT:    msub x0, x8, x9, x2
+; CHECK-NEXT:    umsubl x0, w8, w1, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %and, %zext4
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldr2_ldr2(ptr %x0, ptr %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldr2_ldr2:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    msub x0, x8, x9, x2
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %ext64_2 = load i64, ptr %x1
   %and2 = and i64 %ext64_2, 4294967295
   %mul = mul i64 %and, %and2
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umsubl_ldr2_d(ptr %x0, i64 %x1, i64 %x2) {
 ; CHECK-LABEL: umsubl_ldr2_d:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    and x9, x1, #0xffffffff
-; CHECK-NEXT:    msub x0, x8, x9, x2
+; CHECK-NEXT:    umsubl x0, w8, w9, x2
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 4294967295
   %and2 = and i64 %x1, 4294967295
   %mul = mul i64 %and, %and2
   %sub = sub i64 %x2, %mul
   ret i64 %sub
 }
 
 define i64 @umull_ldr2_w_cc1(ptr %x0, i32 %x1) {
 ; CHECK-LABEL: umull_ldr2_w_cc1:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    mov w8, w1
-; CHECK-NEXT:    and x9, x9, #0x7fffffff
-; CHECK-NEXT:    mul x0, x9, x8
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    and x8, x8, #0x7fffffff
+; CHECK-NEXT:    umull x0, w8, w1
 ; CHECK-NEXT:    ret
 entry:
   %ext64 = load i64, ptr %x0
   %and = and i64 %ext64, 2147483647
   %zext4 = zext i32 %x1 to i64
   %mul = mul i64 %and, %zext4
   ret i64 %mul
 }

llvm/test/CodeGen/AArch64/addcarry-crash.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
 target triple = "arm64-apple-ios7.0"
 
 define i64 @foo(ptr nocapture readonly %ptr, i64 %a, i64 %b, i64 %c) local_unnamed_addr #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    ldr w8, [x0, #4]
 ; CHECK-NEXT:    lsr x9, x1, #32
 ; CHECK-NEXT:    cmn x3, x2
-; CHECK-NEXT:    mul x8, x8, x9
+; CHECK-NEXT:    umull x8, w8, w9
 ; CHECK-NEXT:    cinc x0, x8, hs
 ; CHECK-NEXT:    ret
 entry:
   %0 = lshr i64 %a, 32
   %1 = load i64, ptr %ptr, align 8
   %2 = lshr i64 %1, 32
   %3 = mul nuw i64 %2, %0
   %4 = add i64 %c, %b
   %5 = icmp ult i64 %4, %c
   %6 = zext i1 %5 to i64
   %7 = add i64 %3, %6
   ret i64 %7
 }
 
 attributes #0 = { norecurse nounwind readonly }