[AArch64] Load into zero vector patterns

[dmgreen]

Summary

A LDR will implicitly zero the rest of the vector, so vector_insert(zeros, load, 0) can use a single load. This adds tablegen patterns for both scaled and unscaled loads, detecting where we are inserting a load into the lower element of a zero vector.

[SjoerdMeijer]

Very nice.

[Benoit]

This commit (https://github.com/llvm/llvm-project/commit/83bbd3fdbd75295669cf97967c38810d427c5c25) causes a regression in a downstream project: https://github.com/openxla/iree/issues/12546.

The effect is incorrect results in matrix multiplications, where the result is now filled with zeros instead of the correct, nonzero matrix entries. I will try to debug this some more.

[Benoit]

Minimized end-to-end MLIR testcase here: https://github.com/openxla/iree/pull/12556 :

To make this more helpful, here is:

• the LLVM IR output by our MLIR compiler (ie the input to LLVM aarch64 codegen here): https://gist.github.com/bjacob/2ed1bce14ae4d67b4261adee70089e29
• the good generated aarch64 code (before the regression): https://gist.github.com/bjacob/e69201fc4528ea516fc1788cabb597f0
• the bad generated aarch64 code (after the regression): https://gist.github.com/bjacob/e364981e10e878c4b092728fe6e087aa

[dmgreen]

Hi - thanks for the report. It sounds like the offset might be wrong from the look at the assembly. This instructions specifically:

10534: 40 f4 7f 3d  	ldr	b0, [x2, #4093]
vs
10534: 46 0c 00 d1  	sub	x6, x2, #3
1053c: c0 00 40 0d  	ld1	{ v0.b }[0], [x6]

I think I see the problem - It looks like it should be using an LDUR for those instructions. When printing assembly it will produce an ldr b0, [x2, -3] instruction, but emitting obj files gives the large positive offset. I will put a fix in for that issue now.

I've been unable to produce the same output from https://gist.github.com/bjacob/2ed1bce14ae4d67b4261adee70089e29 though - I probably don't know the right set of commands and was just using mlir-translate to convert the file to llvm-ir. Do you know what commands are needed to compile it to assembly?

[dmgreen]

There is hopefully a fix in 1c6ea961938488997712763762079e535b8b704. Please let me know if that does or doesn't fix your issue, and if you have details on getting assembly from mlir. Thanks

[Benoit]

Thank you very much for the quick fix. I confirm that https://reviews.llvm.org/rG1c6ea961938488997712763762079e535b8b704e fixes the regression.

You probably won't need this anymore since you were able to fix this without it, but just for completeness, here was how to reproduce:

1. Build https://github.com/openxla/iree - following normal build instructions - note that IREE uses its own submodule third_party/llvm-project.
2. Run the IREE compiler from the build directory with these flags:

tools/iree-compile --iree-llvm-target-triple=aarch64-none-linux-android29 --iree-hal-target-backends=llvm-cpu ~/pack_testcase.mlir -o /tmp/a.vmfb --iree-llvm-keep-linker-artifacts

Where the input file pack_testcase.mlir is:

func.func @pack_pad_transpose_1x9xi8_into_2x4x8x4xi8(%arg0 : tensor<1x9xi8>) -> tensor<2x4x8x4xi8> {
  %empty = tensor.empty() : tensor<2x4x8x4xi8>
  %c0_i8 = arith.constant 0 : i8
  %pack = tensor.pack %arg0 padding_value(%c0_i8 : i8) outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [8, 4] into %empty : tensor<1x9xi8> -> tensor<2x4x8x4xi8>
  return %pack : tensor<2x4x8x4xi8>
}

Thanks to the --iree-llvm-keep-linker-artifacts flag, it will print the path to the generated .so, like this

/usr/local/google/home/benoitjacob/pack_testcase.mlir:4:11: remark: linker artifacts for embedded_elf_arm_64 preserved:
    /tmp/pack_pad_transpose_1x9xi8_into_2x4x8x4xi8_dispatch_0-9c98ea.so

So you can then objdump that as usual,

$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-objdump -d /tmp/pack_pad_transpose_1x9xi8_into_2x4x8x4xi8_dispatch_0-9c98ea.so

llvm/lib/Target/AArch64/AArch64InstrInfo.td

def : InstAlias<"ldrsb $Rt, [$Rn, $offset]",
                 (LDURSBXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
 def : InstAlias<"ldrsh $Rt, [$Rn, $offset]",
                 (LDURSHWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
 def : InstAlias<"ldrsh $Rt, [$Rn, $offset]",
                 (LDURSHXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
 def : InstAlias<"ldrsw $Rt, [$Rn, $offset]",
                 (LDURSWi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
 
+// A LDR will implicitly zero the rest of the vector, so vector_insert(zeros,
+// load, 0) can use a single load.
+multiclass LoadInsertZeroPatterns<SDPatternOperator LoadOp, ValueType VT, ValueType HVT, ValueType ScalarVT,
+                                  Instruction LoadInst, Instruction UnscaledLoadInst,
+                                  ComplexPattern Addr, ComplexPattern UnscaledAddr, Operand AddrImm,
+                                  SubRegIndex SubReg> {
+  // Scaled
+  def : Pat <(vector_insert (VT immAllZerosV),
+                (ScalarVT (LoadOp (Addr GPR64sp:$Rn, AddrImm:$offset))), (i64 0)),
+            (SUBREG_TO_REG (i64 0), (LoadInst GPR64sp:$Rn, AddrImm:$offset), SubReg)>;
+  // Unscaled
+  def : Pat <(vector_insert (VT immAllZerosV),
+                 (ScalarVT (LoadOp (UnscaledAddr GPR64sp:$Rn, simm9:$offset))), (i64 0)),
+             (SUBREG_TO_REG (i64 0), (UnscaledLoadInst GPR64sp:$Rn, simm9:$offset), SubReg)>;
+
+  // Half-vector patterns
+  def : Pat <(vector_insert (HVT immAllZerosV),
+                 (ScalarVT (LoadOp (Addr GPR64sp:$Rn, AddrImm:$offset))), (i64 0)),
+             (SUBREG_TO_REG (i64 0), (LoadInst GPR64sp:$Rn, AddrImm:$offset), SubReg)>;
+  // Unscaled
+  def : Pat <(vector_insert (HVT immAllZerosV),
+                 (ScalarVT (LoadOp (UnscaledAddr GPR64sp:$Rn, simm9:$offset))), (i64 0)),
+             (SUBREG_TO_REG (i64 0), (UnscaledLoadInst GPR64sp:$Rn, simm9:$offset), SubReg)>;
+}
+
+defm : LoadInsertZeroPatterns<extloadi8,  v16i8,  v8i8,   i32,  LDRBui, LDRBui,
+                              am_indexed8,  am_unscaled8,  uimm12s1, bsub>;
+defm : LoadInsertZeroPatterns<extloadi16, v8i16,  v4i16,  i32,  LDRHui, LDURHi,
+                              am_indexed16, am_unscaled16, uimm12s2, hsub>;
+defm : LoadInsertZeroPatterns<load,       v4i32,  v2i32,  i32,  LDRSui, LDURSi,
+                              am_indexed32, am_unscaled32, uimm12s4, ssub>;
+defm : LoadInsertZeroPatterns<load,       v2i64,  v1i64,  i64,  LDRDui, LDURDi,
+                              am_indexed64, am_unscaled64, uimm12s8, dsub>;
+defm : LoadInsertZeroPatterns<load,       v8f16,  v4f16,  f16,  LDRHui, LDURHi,
+                              am_indexed16, am_unscaled16, uimm12s2, hsub>;
+defm : LoadInsertZeroPatterns<load,       v8bf16, v4bf16, bf16, LDRHui, LDURHi,
+                              am_indexed16, am_unscaled16, uimm12s2, hsub>;
+defm : LoadInsertZeroPatterns<load,       v4f32,  v2f32,  f32,  LDRSui, LDURSi,
+                              am_indexed32, am_unscaled32, uimm12s4, ssub>;
+defm : LoadInsertZeroPatterns<load,       v2f64,  v1f64,  f64,  LDRDui, LDURDi,
+                              am_indexed64, am_unscaled64, uimm12s8, dsub>;
+
 // Pre-fetch.
 defm PRFUM : PrefetchUnscaled<0b11, 0, 0b10, "prfum",
                   [(AArch64Prefetch timm:$Rt,
                                   (am_unscaled64 GPR64sp:$Rn, simm9:$offset))]>;
 
 //---
 // (unscaled immediate, unprivileged)
 defm LDTRX : LoadUnprivileged<0b11, 0, 0b01, GPR64, "ldtr">;

llvm/test/CodeGen/AArch64/load-insert-zero.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-none-eabi -mattr=+fullfp16,+bf16 | FileCheck %s
 
 define <8 x i8> @loadv8i8(ptr %p) {
 ; CHECK-LABEL: loadv8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr b0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i8, ptr %p
   %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 0
   ret <8 x i8> %v
 }
 
 define <16 x i8> @loadv16i8(ptr %p) {
 ; CHECK-LABEL: loadv16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
+; CHECK-NEXT:    ldr b0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i8, ptr %p
   %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 0
   ret <16 x i8> %v
 }
 
 define <4 x i16> @loadv4i16(ptr %p) {
 ; CHECK-LABEL: loadv4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i16, ptr %p
   %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 0
   ret <4 x i16> %v
 }
 
 define <8 x i16> @loadv8i16(ptr %p) {
 ; CHECK-LABEL: loadv8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i16, ptr %p
   %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 0
   ret <8 x i16> %v
 }
 
 define <2 x i32> @loadv2i32(ptr %p) {
 ; CHECK-LABEL: loadv2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.s }[0], [x0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i32, ptr %p
   %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 0
   ret <2 x i32> %v
 }
 
 define <4 x i32> @loadv4i32(ptr %p) {
 ; CHECK-LABEL: loadv4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.s }[0], [x0]
+; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i32, ptr %p
   %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 0
   ret <4 x i32> %v
 }
 
 define <2 x i64> @loadv2i64(ptr %p) {
 ; CHECK-LABEL: loadv2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.d }[0], [x0]
+; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
   %l = load i64, ptr %p
   %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 0
   ret <2 x i64> %v
 }
 
 
 define <4 x half> @loadv4f16(ptr %p) {
 ; CHECK-LABEL: loadv4f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ret
   %l = load half, ptr %p
   %v = insertelement <4 x half> zeroinitializer, half %l, i32 0
   ret <4 x half> %v
 }
 
 define <8 x half> @loadv8f16(ptr %p) {
 ; CHECK-LABEL: loadv8f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ret
   %l = load half, ptr %p
   %v = insertelement <8 x half> zeroinitializer, half %l, i32 0
   ret <8 x half> %v
 }
 
 define <4 x bfloat> @loadv4bf16(ptr %p) {
 ; CHECK-LABEL: loadv4bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ret
   %l = load bfloat, ptr %p
   %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0
   ret <4 x bfloat> %v
 }
 
 define <8 x bfloat> @loadv8bf16(ptr %p) {
 ; CHECK-LABEL: loadv8bf16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ret
   %l = load bfloat, ptr %p
   %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0
   ret <8 x bfloat> %v
 }
 
 define <2 x float> @loadv2f32(ptr %p) {
 ; CHECK-LABEL: loadv2f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.s }[0], [x0]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ret
   %l = load float, ptr %p
   %v = insertelement <2 x float> zeroinitializer, float %l, i32 0
   ret <2 x float> %v
 }
 
 define <4 x float> @loadv4f32(ptr %p) {
 ; CHECK-LABEL: loadv4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.s }[0], [x0]
+; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ret
   %l = load float, ptr %p
   %v = insertelement <4 x float> zeroinitializer, float %l, i32 0
   ret <4 x float> %v
 }
 
 define <2 x double> @loadv2f64(ptr %p) {
 ; CHECK-LABEL: loadv2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.d }[0], [x0]
+; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
   %l = load double, ptr %p
   %v = insertelement <2 x double> zeroinitializer, double %l, i32 0
   ret <2 x double> %v
 }
 
 
 ; Unscaled
 
 define <8 x i8> @loadv8i8_offset(ptr %p) {
 ; CHECK-LABEL: loadv8i8_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.b }[0], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldr b0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i8, ptr %g
   %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 0
   ret <8 x i8> %v
 }
 
 define <16 x i8> @loadv16i8_offset(ptr %p) {
 ; CHECK-LABEL: loadv16i8_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.b }[0], [x8]
+; CHECK-NEXT:    ldr b0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i8, ptr %g
   %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 0
   ret <16 x i8> %v
 }
 
 define <4 x i16> @loadv4i16_offset(ptr %p) {
 ; CHECK-LABEL: loadv4i16_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldur h0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i16, ptr %g
   %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 0
   ret <4 x i16> %v
 }
 
 define <8 x i16> @loadv8i16_offset(ptr %p) {
 ; CHECK-LABEL: loadv8i16_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
+; CHECK-NEXT:    ldur h0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i16, ptr %g
   %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 0
   ret <8 x i16> %v
 }
 
 define <2 x i32> @loadv2i32_offset(ptr %p) {
 ; CHECK-LABEL: loadv2i32_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.s }[0], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldur s0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i32, ptr %g
   %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 0
   ret <2 x i32> %v
 }
 
 define <4 x i32> @loadv4i32_offset(ptr %p) {
 ; CHECK-LABEL: loadv4i32_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.s }[0], [x8]
+; CHECK-NEXT:    ldur s0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i32, ptr %g
   %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 0
   ret <4 x i32> %v
 }
 
 define <2 x i64> @loadv2i64_offset(ptr %p) {
 ; CHECK-LABEL: loadv2i64_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.d }[0], [x8]
+; CHECK-NEXT:    ldur d0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load i64, ptr %g
   %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 0
   ret <2 x i64> %v
 }
 
 
 define <4 x half> @loadv4f16_offset(ptr %p) {
 ; CHECK-LABEL: loadv4f16_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldur h0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load half, ptr %g
   %v = insertelement <4 x half> zeroinitializer, half %l, i32 0
   ret <4 x half> %v
 }
 
 define <8 x half> @loadv8f16_offset(ptr %p) {
 ; CHECK-LABEL: loadv8f16_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
+; CHECK-NEXT:    ldur h0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load half, ptr %g
   %v = insertelement <8 x half> zeroinitializer, half %l, i32 0
   ret <8 x half> %v
 }
 
 define <4 x bfloat> @loadv4bf16_offset(ptr %p) {
 ; CHECK-LABEL: loadv4bf16_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldur h0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load bfloat, ptr %g
   %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0
   ret <4 x bfloat> %v
 }
 
 define <8 x bfloat> @loadv8bf16_offset(ptr %p) {
 ; CHECK-LABEL: loadv8bf16_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
+; CHECK-NEXT:    ldur h0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load bfloat, ptr %g
   %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0
   ret <8 x bfloat> %v
 }
 
 define <2 x float> @loadv2f32_offset(ptr %p) {
 ; CHECK-LABEL: loadv2f32_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.s }[0], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ldur s0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load float, ptr %g
   %v = insertelement <2 x float> zeroinitializer, float %l, i32 0
   ret <2 x float> %v
 }
 
 define <4 x float> @loadv4f32_offset(ptr %p) {
 ; CHECK-LABEL: loadv4f32_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.s }[0], [x8]
+; CHECK-NEXT:    ldur s0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load float, ptr %g
   %v = insertelement <4 x float> zeroinitializer, float %l, i32 0
   ret <4 x float> %v
 }
 
 define <2 x double> @loadv2f64_offset(ptr %p) {
 ; CHECK-LABEL: loadv2f64_offset:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.d }[0], [x8]
+; CHECK-NEXT:    ldur d0, [x0, #1]
 ; CHECK-NEXT:    ret
   %g = getelementptr inbounds i8, ptr %p, i64 1
   %l = load double, ptr %g
   %v = insertelement <2 x double> zeroinitializer, double %l, i32 0
   ret <2 x double> %v
 }
 
 
 define void @predictor_4x4_neon(ptr nocapture noundef writeonly %0, i64 noundef %1, ptr nocapture noundef readonly %2, ptr nocapture noundef readnone %3) {
 ; CHECK-LABEL: predictor_4x4_neon:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    ldur w9, [x2, #2]
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x2, #1
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.s }[0], [x2]
-; CHECK-NEXT:    mov v1.s[0], w9
-; CHECK-NEXT:    ld1 { v2.s }[0], [x8]
-; CHECK-NEXT:    lsr w8, w9, #24
-; CHECK-NEXT:    uaddl v1.8h, v1.8b, v0.8b
-; CHECK-NEXT:    dup v4.8b, w8
-; CHECK-NEXT:    ushll v3.8h, v2.8b, #1
+; CHECK-NEXT:    ldr s1, [x2]
 ; CHECK-NEXT:    lsl x8, x1, #1
-; CHECK-NEXT:    urhadd v0.8b, v0.8b, v2.8b
+; CHECK-NEXT:    ldur s2, [x2, #1]
+; CHECK-NEXT:    mov v0.s[0], w9
+; CHECK-NEXT:    lsr w9, w9, #24
+; CHECK-NEXT:    ushll v3.8h, v2.8b, #1
+; CHECK-NEXT:    dup v4.8b, w9
 ; CHECK-NEXT:    add x9, x8, x1
-; CHECK-NEXT:    add v1.8h, v1.8h, v3.8h
-; CHECK-NEXT:    zip1 v2.2s, v0.2s, v4.2s
-; CHECK-NEXT:    str s0, [x0]
-; CHECK-NEXT:    rshrn v1.8b, v1.8h, #2
-; CHECK-NEXT:    str s1, [x0, x1]
-; CHECK-NEXT:    zip1 v3.2s, v1.2s, v4.2s
-; CHECK-NEXT:    ext v0.8b, v2.8b, v0.8b, #1
-; CHECK-NEXT:    str s0, [x0, x8]
+; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    urhadd v1.8b, v1.8b, v2.8b
+; CHECK-NEXT:    str s1, [x0]
+; CHECK-NEXT:    add v0.8h, v0.8h, v3.8h
+; CHECK-NEXT:    zip1 v2.2s, v1.2s, v4.2s
+; CHECK-NEXT:    rshrn v0.8b, v0.8h, #2
+; CHECK-NEXT:    str s0, [x0, x1]
+; CHECK-NEXT:    zip1 v3.2s, v0.2s, v4.2s
+; CHECK-NEXT:    ext v1.8b, v2.8b, v0.8b, #1
+; CHECK-NEXT:    str s1, [x0, x8]
 ; CHECK-NEXT:    ext v2.8b, v3.8b, v0.8b, #1
 ; CHECK-NEXT:    str s2, [x0, x9]
 ; CHECK-NEXT:    ret
   %5 = load i32, ptr %2, align 4
   %6 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %5, i64 0
   %7 = bitcast <2 x i32> %6 to <8 x i8>
   %8 = getelementptr inbounds i8, ptr %2, i64 1
   %9 = load i32, ptr %8, align 4
; CHECK-NEXT:    ret
   %47 = extractelement <2 x i32> %44, i64 0
   store i32 %47, ptr %46, align 4
   ret void
 }
 
 define void @predictor_4x4_neon_new(ptr nocapture noundef writeonly %0, i64 noundef %1, ptr nocapture noundef readonly %2, ptr nocapture noundef readnone %3) {
 ; CHECK-LABEL: predictor_4x4_neon_new:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    add x8, x2, #1
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    add x9, x2, #2
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    add x10, x2, #3
-; CHECK-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-NEXT:    ld1 { v0.s }[0], [x2]
-; CHECK-NEXT:    ld1 { v1.s }[0], [x8]
+; CHECK-NEXT:    ldr s0, [x2]
 ; CHECK-NEXT:    lsl x8, x1, #1
-; CHECK-NEXT:    ld1 { v2.s }[0], [x9]
+; CHECK-NEXT:    ldur s1, [x2, #1]
 ; CHECK-NEXT:    add x9, x8, x1
-; CHECK-NEXT:    ld1 { v3.s }[0], [x10]
+; CHECK-NEXT:    ldur s2, [x2, #2]
+; CHECK-NEXT:    ldur s3, [x2, #3]
 ; CHECK-NEXT:    uaddl v4.8h, v1.8b, v0.8b
 ; CHECK-NEXT:    urhadd v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    uaddl v5.8h, v2.8b, v1.8b
 ; CHECK-NEXT:    uaddl v3.8h, v3.8b, v2.8b
 ; CHECK-NEXT:    str s0, [x0]
-; CHECK-NEXT:    urhadd v1.8b, v1.8b, v2.8b
 ; CHECK-NEXT:    add v4.8h, v4.8h, v5.8h
 ; CHECK-NEXT:    add v3.8h, v3.8h, v5.8h
-; CHECK-NEXT:    rshrn v4.8b, v4.8h, #2
-; CHECK-NEXT:    rshrn v0.8b, v3.8h, #2
-; CHECK-NEXT:    str s4, [x0, x1]
-; CHECK-NEXT:    str s1, [x0, x8]
-; CHECK-NEXT:    str s0, [x0, x9]
+; CHECK-NEXT:    rshrn v0.8b, v4.8h, #2
+; CHECK-NEXT:    str s0, [x0, x1]
+; CHECK-NEXT:    urhadd v0.8b, v1.8b, v2.8b
+; CHECK-NEXT:    rshrn v1.8b, v3.8h, #2
+; CHECK-NEXT:    str s0, [x0, x8]
+; CHECK-NEXT:    str s1, [x0, x9]
 ; CHECK-NEXT:    ret
   %5 = load i32, ptr %2, align 4
   %6 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %5, i64 0
   %7 = bitcast <2 x i32> %6 to <8 x i8>
   %8 = getelementptr inbounds i8, ptr %2, i64 1
   %9 = load i32, ptr %8, align 4
   %10 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %9, i64 0
   %11 = bitcast <2 x i32> %10 to <8 x i8>

llvm/test/CodeGen/AArch64/speculation-hardening-loads.ll

 ; CHECK-NEXT:  ret
 }
 
 define <2 x double> @f_usedefvectorload(ptr %a, ptr %b) speculative_load_hardening {
 entry:
 ; CHECK-LABEL: f_usedefvectorload
 ; CHECK:       cmp     sp, #0
 ; CHECK-NEXT:  csetm   x16, ne
-; CHECK-NEXT:  movi    v0.2d, #0000000000000000
 ; CHECK-NEXT:  and     x1, x1, x16
 ; CHECK-NEXT:  csdb
 ; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
-; CHECK-NEXT:  ld1     { v0.d }[0], [x1]
+; CHECK-NEXT:  ldr     d0, [x1]
 ; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
 ; CHECK-NEXT:  mov     sp, [[TMPREG]]
 ; CHECK-NEXT:  ret
   %0 = load double, ptr %b, align 16
   %vld1_lane = insertelement <2 x double> <double undef, double 0.000000e+00>, double %0, i32 0
   ret <2 x double> %vld1_lane
 }