[AArch64] Avoid unnecessary vector byte-swapping in big-endian

[pbarrio]

Summary

Loads/stores of some NEON vector types are promoted to other vector
types with different lane sizes but same vector size. This is not a
problem in little-endian but, when in big-endian, it requires
additional byte reversals required to preserve the lane ordering
while keeping the right endianness of the data inside each lane.
For example:

%1 = load <4 x half>, <4 x half>* %p

results in the following assembly:

ld1 { v0.2s }, [x1]
rev32 v0.4h, v0.4h

This patch changes the promotion of these loads/stores so that the
actual vector load/store (LD1/ST1) takes care of the endianness
correctly and there is no need for further byte reversals. The
previous code now results in the following assembly:

ld1 { v0.4h }, [x1]

[efriedma]

lib/Target/AArch64/AArch64ISelLowering.cpp

726 ↗

(On Diff #130376)

Could you use changeVectorElementTypeToInteger or something like that rather than listing out all the types individually?

743 ↗

(On Diff #130376)

Missing break.

[pbarrio]

Instead of an ugly switch, reuse already-existing functionality in LLVM to do
the conversion of FP vector types into int vector types.

[pbarrio]

Thanks for the suggestion, very helpful :)

[efriedma]

LGTM with one minor tweak.

lib/Target/AArch64/AArch64ISelLowering.cpp

724 ↗

(On Diff #130864)

We can assume VT is a vector type here.

[pbarrio]

VT should be a vector in addTypeForNEON, so the check can be an assert
instead of a proper test in the conditional.

[pbarrio]

Waiting a bit just in case there's extra feedback for the last change. I will commit this patch later today.

[MatzeB]

This apparently defeats the logic in splitStores() (aka the infamous FeatureMisalignedSlow128Bit) because that code had an exception for v2i64 vector types but wants to split all other vector types. An internal test case of ours:

define void @test_split_16B(<4 x float> %val, <4 x float>* %addr) {
  store <4 x float> %val, <4 x float>* %addr, align 8
  ret void
}

would no longer show the store getting split.

Independently of this I disabled FeatureMisalignedSlow128 for apple a few hours later so my interest is limited now :)

But I think Exynos is still using the "feature", so they may want to look into this.

llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp

for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
       setOperationAction(ISD::FROUND, Ty, Legal);
     }
   }
 
   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 }
 
 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
-  if (VT == MVT::v2f32 || VT == MVT::v4f16) {
-    setOperationAction(ISD::LOAD, VT, Promote);
-    AddPromotedToType(ISD::LOAD, VT, MVT::v2i32);
-
-    setOperationAction(ISD::STORE, VT, Promote);
-    AddPromotedToType(ISD::STORE, VT, MVT::v2i32);
-  } else if (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16) {
-    setOperationAction(ISD::LOAD, VT, Promote);
-    AddPromotedToType(ISD::LOAD, VT, MVT::v2i64);
+  assert(VT.isVector() && "VT should be a vector type");
 
-    setOperationAction(ISD::STORE, VT, Promote);
-    AddPromotedToType(ISD::STORE, VT, MVT::v2i64);
+  if (VT.isFloatingPoint()) {
+    MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
+    setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
+    setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
   }
 
   // Mark vector float intrinsics as expand.
   if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
     setOperationAction(ISD::FSIN, VT, Expand);
     setOperationAction(ISD::FCOS, VT, Expand);
     setOperationAction(ISD::FPOW, VT, Expand);
     setOperationAction(ISD::FLOG, VT, Expand);

llvm/trunk/test/CodeGen/AArch64/arm64-big-endian-bitconverts.ll

; CHECK: str
     %3 = bitcast <2 x i32> %2 to i64
     %4 = add i64 %3, %3
     store i64 %4, i64* %q
     ret void
 }
 
 ; CHECK-LABEL: test_i64_v4f16:
 define void @test_i64_v4f16(<4 x half>* %p, i64* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2s }
-; CHECK: rev32 v{{[0-9]+}}.4h
+; CHECK: ld1 { v{{[0-9]+}}.4h }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev64 v{{[0-9]+}}.4h
 ; CHECK: str
     %1 = load <4 x half>, <4 x half>* %p
     %2 = fadd <4 x half> %1, %1
     %3 = bitcast <4 x half> %2 to i64
     %4 = add i64 %3, %3
     store i64 %4, i64* %q
     ret void
; CHECK: str
     %3 = bitcast <4 x i16> %2 to double
     %4 = fadd double %3, %3
     store double %4, double* %q
     ret void
 }
 
 ; CHECK-LABEL: test_f64_v4f16:
 define void @test_f64_v4f16(<4 x half>* %p, double* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2s }
-; CHECK: rev32 v{{[0-9]+}}.4h
+; CHECK: ld1 { v{{[0-9]+}}.4h }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev64 v{{[0-9]+}}.4h
+; CHECK: fadd
 ; CHECK: str
     %1 = load <4 x half>, <4 x half>* %p
     %2 = fadd <4 x half> %1, %1
     %3 = bitcast <4 x half> %2 to double
     %4 = fadd double %3, %3
     store double %4, double* %q
     ret void
 }
; CHECK: str
     %3 = bitcast <2 x i32> %2 to <1 x i64>
     %4 = add <1 x i64> %3, %3
     store <1 x i64> %4, <1 x i64>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v1i64_v4f16:
 define void @test_v1i64_v4f16(<4 x half>* %p, <1 x i64>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2s }
-; CHECK: rev32 v{{[0-9]+}}.4h
+; CHECK: ld1 { v{{[0-9]+}}.4h }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev64 v{{[0-9]+}}.4h
 ; CHECK: str
     %1 = load <4 x half>, <4 x half>* %p
     %2 = fadd <4 x half> %1, %1
     %3 = bitcast <4 x half> %2 to <1 x i64>
     %4 = add <1 x i64> %3, %3
     store <1 x i64> %4, <1 x i64>* %q
     ret void
; CHECK: st1 { v{{[0-9]+}}.2s }
     %3 = bitcast <4 x i16> %2 to <2 x float>
     %4 = fadd <2 x float> %3, %3
     store <2 x float> %4, <2 x float>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v2f32_v4f16:
 define void @test_v2f32_v4f16(<4 x half>* %p, <2 x float>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2s }
-; CHECK: rev32 v{{[0-9]+}}.4h
+; CHECK: ld1 { v{{[0-9]+}}.4h }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev32 v{{[0-9]+}}.4h
 ; CHECK: st1 { v{{[0-9]+}}.2s }
     %1 = load <4 x half>, <4 x half>* %p
     %2 = fadd <4 x half> %1, %1
     %3 = bitcast <4 x half> %2 to <2 x float>
     %4 = fadd <2 x float> %3, %3
     store <2 x float> %4, <2 x float>* %q
     ret void
; CHECK: st1 { v{{[0-9]+}}.4h }
     %3 = bitcast <2 x i32> %2 to <4 x i16>
     %4 = add <4 x i16> %3, %3
     store <4 x i16> %4, <4 x i16>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4i16_v4f16:
 define void @test_v4i16_v4f16(<4 x half>* %p, <4 x i16>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2s }
-; CHECK: rev32 v{{[0-9]+}}.4h
+; CHECK: ld1 { v{{[0-9]+}}.4h }
+; CHECK-NOT: rev
 ; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load <4 x half>, <4 x half>* %p
     %2 = fadd <4 x half> %1, %1
     %3 = bitcast <4 x half> %2 to <4 x i16>
     %4 = add <4 x i16> %3, %3
     store <4 x i16> %4, <4 x i16>* %q
     ret void
 }
; CHECK: st1 { v{{[0-9]+}}.4h }
     store <4 x i16> %4, <4 x i16>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_i64:
 define void @test_v4f16_i64(i64* %p, <4 x half>* %q) {
 ; CHECK: ldr
 ; CHECK: rev64 v{{[0-9]+}}.4h
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load i64, i64* %p
     %2 = add i64 %1, %1
     %3 = bitcast i64 %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_f64:
 define void @test_v4f16_f64(double* %p, <4 x half>* %q) {
 ; CHECK: ldr
 ; CHECK: rev64 v{{[0-9]+}}.4h
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load double, double* %p
     %2 = fadd double %1, %1
     %3 = bitcast double %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_v1i64:
 define void @test_v4f16_v1i64(<1 x i64>* %p, <4 x half>* %q) {
 ; CHECK: ldr
 ; CHECK: rev64 v{{[0-9]+}}.4h
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load <1 x i64>, <1 x i64>* %p
     %2 = add <1 x i64> %1, %1
     %3 = bitcast <1 x i64> %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_v2f32:
 define void @test_v4f16_v2f32(<2 x float>* %p, <4 x half>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.2s }
 ; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load <2 x float>, <2 x float>* %p
     %2 = fadd <2 x float> %1, %1
     %3 = bitcast <2 x float> %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_v2i32:
 define void @test_v4f16_v2i32(<2 x i32>* %p, <4 x half>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.2s }
 ; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load <2 x i32>, <2 x i32>* %p
     %2 = add <2 x i32> %1, %1
     %3 = bitcast <2 x i32> %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_v4i16:
 define void @test_v4f16_v4i16(<4 x i16>* %p, <4 x half>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.4h }
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load <4 x i16>, <4 x i16>* %p
     %2 = add <4 x i16> %1, %1
     %3 = bitcast <4 x i16> %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_v8i8:
 define void @test_v4f16_v8i8(<8 x i8>* %p, <4 x half>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.8b }
 ; CHECK: rev16 v{{[0-9]+}}.8b
-; CHECK: rev32 v{{[0-9]+}}.4h
-; CHECK: st1 { v{{[0-9]+}}.2s }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4h }
     %1 = load <8 x i8>, <8 x i8>* %p
     %2 = add <8 x i8> %1, %1
     %3 = bitcast <8 x i8> %2 to <4 x half>
     %4 = fadd <4 x half> %3, %3
     store <4 x half> %4, <4 x half>* %q
     ret void
 }
 
; CHECK: str
     %3 = bitcast <2 x i64> %2 to fp128
     %4 = fadd fp128 %3, %3
     store fp128 %4, fp128* %q
     ret void
 }
 
 ; CHECK-LABEL: test_f128_v4f32:
 define void @test_f128_v4f32(<4 x float>* %p, fp128* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: ld1 { v{{[0-9]+}}.4s }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev64 v{{[0-9]+}}.4s
 ; CHECK: ext
 ; CHECK: str q
     %1 = load <4 x float>, <4 x float>* %p
     %2 = fadd <4 x float> %1, %1
     %3 = bitcast <4 x float> %2 to fp128
     %4 = fadd fp128 %3, %3
     store fp128 %4, fp128* %q
; CHECK: st1 { v{{[0-9]+}}.2d }
     %3 = bitcast <2 x i64> %2 to <2 x double>
     %4 = fadd <2 x double> %3, %3
     store <2 x double> %4, <2 x double>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v2f64_v4f32:
 define void @test_v2f64_v4f32(<4 x float>* %p, <2 x double>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: ld1 { v{{[0-9]+}}.4s }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev64 v{{[0-9]+}}.4s
 ; CHECK: st1 { v{{[0-9]+}}.2d }
     %1 = load <4 x float>, <4 x float>* %p
     %2 = fadd <4 x float> %1, %1
     %3 = bitcast <4 x float> %2 to <2 x double>
     %4 = fadd <2 x double> %3, %3
     store <2 x double> %4, <2 x double>* %q
     ret void
; CHECK: st1 { v{{[0-9]+}}.2d }
     %3 = bitcast <2 x double> %2 to <2 x i64>
     %4 = add <2 x i64> %3, %3
     store <2 x i64> %4, <2 x i64>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v2i64_v4f32:
 define void @test_v2i64_v4f32(<4 x float>* %p, <2 x i64>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: ld1 { v{{[0-9]+}}.4s }
+; CHECK-NOT: rev
+; CHECK: fadd
 ; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: add
 ; CHECK: st1 { v{{[0-9]+}}.2d }
     %1 = load <4 x float>, <4 x float>* %p
     %2 = fadd <4 x float> %1, %1
     %3 = bitcast <4 x float> %2 to <2 x i64>
     %4 = add <2 x i64> %3, %3
     store <2 x i64> %4, <2 x i64>* %q
     ret void
 }
; CHECK: st1 { v{{[0-9]+}}.2d }
     ret void
 }
 
 ; CHECK-LABEL: test_v4f32_f128:
 define void @test_v4f32_f128(fp128* %p, <4 x float>* %q) {
 ; CHECK: ldr q
 ; CHECK: rev64 v{{[0-9]+}}.4s
 ; CHECK: ext
-; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: st1 { v{{[0-9]+}}.2d }
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load fp128, fp128* %p
     %2 = fadd fp128 %1, %1
     %3 = bitcast fp128 %2 to <4 x float>
     %4 = fadd <4 x float> %3, %3
     store <4 x float> %4, <4 x float>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f32_v2f64:
 define void @test_v4f32_v2f64(<2 x double>* %p, <4 x float>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.2d }
 ; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: st1 { v{{[0-9]+}}.2d }
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load <2 x double>, <2 x double>* %p
     %2 = fadd <2 x double> %1, %1
     %3 = bitcast <2 x double> %2 to <4 x float>
     %4 = fadd <4 x float> %3, %3
     store <4 x float> %4, <4 x float>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f32_v2i64:
 define void @test_v4f32_v2i64(<2 x i64>* %p, <4 x float>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.2d }
 ; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: st1 { v{{[0-9]+}}.2d }
+; CHECK: fadd
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load <2 x i64>, <2 x i64>* %p
     %2 = add <2 x i64> %1, %1
     %3 = bitcast <2 x i64> %2 to <4 x float>
     %4 = fadd <4 x float> %3, %3
     store <4 x float> %4, <4 x float>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f32_v4i32:
 define void @test_v4f32_v4i32(<4 x i32>* %p, <4 x float>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.4s }
-; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: st1 { v{{[0-9]+}}.2d }
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load <4 x i32>, <4 x i32>* %p
     %2 = add <4 x i32> %1, %1
     %3 = bitcast <4 x i32> %2 to <4 x float>
     %4 = fadd <4 x float> %3, %3
     store <4 x float> %4, <4 x float>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f32_v8i16:
 define void @test_v4f32_v8i16(<8 x i16>* %p, <4 x float>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.8h }
 ; CHECK: rev32 v{{[0-9]+}}.8h
-; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: st1 { v{{[0-9]+}}.2d }
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load <8 x i16>, <8 x i16>* %p
     %2 = add <8 x i16> %1, %1
     %3 = bitcast <8 x i16> %2 to <4 x float>
     %4 = fadd <4 x float> %3, %3
     store <4 x float> %4, <4 x float>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f32_v16i8:
 define void @test_v4f32_v16i8(<16 x i8>* %p, <4 x float>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.16b }
 ; CHECK: rev32 v{{[0-9]+}}.16b
-; CHECK: rev64 v{{[0-9]+}}.4s
-; CHECK: st1 { v{{[0-9]+}}.2d }
+; CHECK-NOT: rev
+; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load <16 x i8>, <16 x i8>* %p
     %2 = add <16 x i8> %1, %1
     %3 = bitcast <16 x i8> %2 to <4 x float>
     %4 = fadd <4 x float> %3, %3
     store <4 x float> %4, <4 x float>* %q
     ret void
 }
 
; CHECK: st1 { v{{[0-9]+}}.4s }
     %3 = bitcast <2 x i64> %2 to <4 x i32>
     %4 = add <4 x i32> %3, %3
     store <4 x i32> %4, <4 x i32>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4i32_v4f32:
 define void @test_v4i32_v4f32(<4 x float>* %p, <4 x i32>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: ld1 { v{{[0-9]+}}.4s }
+; CHECK-NOT: rev
 ; CHECK: st1 { v{{[0-9]+}}.4s }
     %1 = load <4 x float>, <4 x float>* %p
     %2 = fadd <4 x float> %1, %1
     %3 = bitcast <4 x float> %2 to <4 x i32>
     %4 = add <4 x i32> %3, %3
     store <4 x i32> %4, <4 x i32>* %q
     ret void
 }
; CHECK: st1 { v{{[0-9]+}}.8h }
     %3 = bitcast <2 x i64> %2 to <8 x i16>
     %4 = add <8 x i16> %3, %3
     store <8 x i16> %4, <8 x i16>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v8i16_v4f32:
 define void @test_v8i16_v4f32(<4 x float>* %p, <8 x i16>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: ld1 { v{{[0-9]+}}.4s }
 ; CHECK: rev32 v{{[0-9]+}}.8h
+; CHECK-NOT: rev
 ; CHECK: st1 { v{{[0-9]+}}.8h }
     %1 = load <4 x float>, <4 x float>* %p
     %2 = fadd <4 x float> %1, %1
     %3 = bitcast <4 x float> %2 to <8 x i16>
     %4 = add <8 x i16> %3, %3
     store <8 x i16> %4, <8 x i16>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v8i16_v4i32:
 define void @test_v8i16_v4i32(<4 x i32>* %p, <8 x i16>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.4s }
 ; CHECK: rev32 v{{[0-9]+}}.8h
 ; CHECK: st1 { v{{[0-9]+}}.8h }
     %1 = load <4 x i32>, <4 x i32>* %p
     %2 = add <4 x i32> %1, %1
     %3 = bitcast <4 x i32> %2 to <8 x i16>
     %4 = add <8 x i16> %3, %3
     store <8 x i16> %4, <8 x i16>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v8i16_v8f16:
 define void @test_v8i16_v8f16(<8 x half>* %p, <8 x i16>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.8h
+; CHECK: ld1 { v{{[0-9]+}}.8h }
+; CHECK-NOT: rev
 ; CHECK: st1 { v{{[0-9]+}}.8h }
     %1 = load <8 x half>, <8 x half>* %p
     %2 = fadd <8 x half> %1, %1
     %3 = bitcast <8 x half> %2 to <8 x i16>
     %4 = add <8 x i16> %3, %3
     store <8 x i16> %4, <8 x i16>* %q
     ret void
 }
; CHECK: st1 { v{{[0-9]+}}.16b }
     %3 = bitcast <2 x i64> %2 to <16 x i8>
     %4 = add <16 x i8> %3, %3
     store <16 x i8> %4, <16 x i8>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v16i8_v4f32:
 define void @test_v16i8_v4f32(<4 x float>* %p, <16 x i8>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.4s
+; CHECK: ld1 { v{{[0-9]+}}.4s }
 ; CHECK: rev32 v{{[0-9]+}}.16b
+; CHECK-NOT: rev
 ; CHECK: st1 { v{{[0-9]+}}.16b }
     %1 = load <4 x float>, <4 x float>* %p
     %2 = fadd <4 x float> %1, %1
     %3 = bitcast <4 x float> %2 to <16 x i8>
     %4 = add <16 x i8> %3, %3
     store <16 x i8> %4, <16 x i8>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v16i8_v4i32:
 define void @test_v16i8_v4i32(<4 x i32>* %p, <16 x i8>* %q) {
 ; CHECK: ld1 { v{{[0-9]+}}.4s }
 ; CHECK: rev32 v{{[0-9]+}}.16b
 ; CHECK: st1 { v{{[0-9]+}}.16b }
     %1 = load <4 x i32>, <4 x i32>* %p
     %2 = add <4 x i32> %1, %1
     %3 = bitcast <4 x i32> %2 to <16 x i8>
     %4 = add <16 x i8> %3, %3
     store <16 x i8> %4, <16 x i8>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v16i8_v8f16:
 define void @test_v16i8_v8f16(<8 x half>* %p, <16 x i8>* %q) {
-; CHECK: ld1 { v{{[0-9]+}}.2d }
-; CHECK: rev64 v{{[0-9]+}}.8h
+; CHECK: ld1 { v{{[0-9]+}}.8h }
 ; CHECK: rev16 v{{[0-9]+}}.16b
+; CHECK-NOT: rev
 ; CHECK: st1 { v{{[0-9]+}}.16b }
     %1 = load <8 x half>, <8 x half>* %p
     %2 = fadd <8 x half> %1, %1
     %3 = bitcast <8 x half> %2 to <16 x i8>
     %4 = add <16 x i8> %3, %3
     store <16 x i8> %4, <16 x i8>* %q
     ret void
 }
; CHECK: st1 { v{{[0-9]+}}.16b }
     store <16 x i8> %4, <16 x i8>* %q
     ret void
 }
 
 ; CHECK-LABEL: test_v4f16_struct:
 %struct.struct1 = type { half, half, half, half }
 define %struct.struct1 @test_v4f16_struct(%struct.struct1* %ret) {
 entry:
-; CHECK: ld1 { {{v[0-9]+}}.2s }
-; CHECK: rev32
-; CHECK-NOT; rev64
+; CHECK: ld1 { {{v[0-9]+}}.4h }
+; CHECK-NOT: rev
   %0 = bitcast %struct.struct1* %ret to <4 x half>*
   %1 = load <4 x half>, <4 x half>* %0, align 2
   %2 = extractelement <4 x half> %1, i32 0
   %.fca.0.insert = insertvalue %struct.struct1 undef, half %2, 0
   ret %struct.struct1 %.fca.0.insert
 }