[ARM,MVE] Add intrinsics for vector get/set lane.

[simon_tatham]

Summary

This adds the vgetq_lane and vsetq_lane families, to copy between
a scalar and a specified lane of a vector.

One of the new vgetq_lane intrinsics returns a float16_t, which
causes a compile error if %clang_cc1 doesn't get the option
-fallow-half-arguments-and-returns. The driver passes that option to
cc1 already, but I've had to edit all the explicit cc1 command lines
in the existing MVE intrinsics tests.

A couple of fixes are included for the code I wrote up front in
MveEmitter to support lane-index immediates (and which nothing has
tested until now): the type was wrong (uint32_t instead of int)
and the range was off by one.

I've also added a method of bypassing the default promotion to i32
that is done by the MveEmitter code generation: it's sensible to
promote short scalars like i16 to i32 if they're going to be
passed to custom IR intrinsics representing a machine instruction
operating on GPRs, but not if they're going to be passed to standard
IR operations like insertelement which expect the exact type.

[dmgreen]

Thanks. LGTM.

clang/include/clang/Basic/arm_mve.td

let params = T.All in {
   // full name never appears in the header file due to the polymorphicOnly
   // flag, and the _polymorphic suffix is omitted from the shortened name by
   // the custom PolymorphicNameType here.
   let polymorphicOnly = 1, nonEvaluating = 1,
       pnt = PolymorphicNameType<1, "polymorphic"> in {
     def vuninitializedq_polymorphic: Intrinsic<
         Vector, (args Vector), (undef Vector)>;
   }
+
+  def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane),
+                            (xelt_var $v, $lane)>;
+  def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane),
+                            (ielt_var $v, $e, $lane)>;
 }

clang/include/clang/Basic/arm_mve_defs.td

def store: IRBuilder<"CreateStore"> {
   let special_params = [IRBuilderAddrParam<1>];
 }
 def xval: IRBuilder<"CreateExtractValue"> {
   let special_params = [IRBuilderIntParam<1, "unsigned">];
 }
 def ielt_const: IRBuilder<"CreateInsertElement"> {
   let special_params = [IRBuilderIntParam<2, "uint64_t">];
 }
+def ielt_var: IRBuilder<"CreateInsertElement">;
+def xelt_var: IRBuilder<"CreateExtractElement">;
 def trunc: IRBuilder<"CreateTrunc">;
 def bitcast: IRBuilder<"CreateBitCast">;
 def extend: CGHelperFn<"SignOrZeroExtend"> {
   let special_params = [IRBuilderIntParam<2, "bool">];
 }
 def zeroinit: IRFunction<"llvm::Constant::getNullValue">;
 def undef: IRFunction<"UndefValue::get">;
 

 // Instances of Type intended to be used directly in the specification of an
 // intrinsic in arm_mve.td.
 
 // The type Void can be used for the return type of an intrinsic, and as the
 // parameter type for intrinsics that aren't actually parameterised by any kind
 // of _s32 / _f16 / _u8 suffix.
 def Void : Type;
 
+// A wrapper you can put on an intrinsic's argument type to prevent it from
+// being automatically promoted to i32 from a smaller integer type.
+class unpromoted<Type t> : Type { Type underlying_type = t; }
+
 // Primitive types: base class, and an instance for the set of scalar integer
 // and floating types that MVE uses.
 class PrimitiveType<string kind_, int size_>: Type {
   string kind = kind_;
   int size = size_;
   string nameOverride = "";
 }
 

 //
 // imm_0toNm1 is the same but with the range offset by 1, i.e. 0 to N-1
 // inclusive.
 def imm_1toN : Immediate<u32, IB_EltBit<1>>;
 def imm_0toNm1 : Immediate<u32, IB_EltBit<0>>;
 
 // imm_lane has to be the index of a vector lane in the main vector type, i.e
 // it can range from 0 to (128 / size of scalar)-1 inclusive. (e.g. vgetq_lane)
-def imm_lane : Immediate<u32, IB_LaneIndex>;
+def imm_lane : Immediate<sint, IB_LaneIndex>;
 
 // imm_1to32 can be in the range 1 to 32, unconditionally. (e.g. scalar shift
 // intrinsics)
 def imm_1to32 : Immediate<u32, IB_ConstRange<1, 32>>;
 
 // imm_1248 can be 1, 2, 4 or 8. (e.g. vidupq)
 def imm_1248 : Immediate<u32, IB_ConstRange<1, 8>> {
   let extra = "Power2";

clang/test/CodeGen/arm-mve-intrinsics/get-set-lane.c

+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg -sroa -early-cse | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg -sroa -early-cse | FileCheck %s
+
+#include <arm_mve.h>
+
+// CHECK-LABEL: @test_vgetq_lane_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x half> [[A:%.*]], i32 2
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[TMP0]] to i16
+// CHECK-NEXT:    [[TMP_0_INSERT_EXT:%.*]] = zext i16 [[TMP1]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP_0_INSERT_EXT]] to float
+// CHECK-NEXT:    ret float [[TMP2]]
+//
+float16_t test_vgetq_lane_f16(float16x8_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 2);
+#else /* POLYMORPHIC */
+    return vgetq_lane_f16(a, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 3
+// CHECK-NEXT:    ret float [[TMP0]]
+//
+float32_t test_vgetq_lane_f32(float32x4_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 3);
+#else /* POLYMORPHIC */
+    return vgetq_lane_f32(a, 3);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i16> [[A:%.*]], i32 4
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+int16_t test_vgetq_lane_s16(int16x8_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 4);
+#else /* POLYMORPHIC */
+    return vgetq_lane_s16(a, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+int32_t test_vgetq_lane_s32(int32x4_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 0);
+#else /* POLYMORPHIC */
+    return vgetq_lane_s32(a, 0);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[A:%.*]], i32 0
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+int64_t test_vgetq_lane_s64(int64x2_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 0);
+#else /* POLYMORPHIC */
+    return vgetq_lane_s64(a, 0);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <16 x i8> [[A:%.*]], i32 10
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+int8_t test_vgetq_lane_s8(int8x16_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 10);
+#else /* POLYMORPHIC */
+    return vgetq_lane_s8(a, 10);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i16> [[A:%.*]], i32 3
+// CHECK-NEXT:    ret i16 [[TMP0]]
+//
+uint16_t test_vgetq_lane_u16(uint16x8_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 3);
+#else /* POLYMORPHIC */
+    return vgetq_lane_u16(a, 3);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 3
+// CHECK-NEXT:    ret i32 [[TMP0]]
+//
+uint32_t test_vgetq_lane_u32(uint32x4_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 3);
+#else /* POLYMORPHIC */
+    return vgetq_lane_u32(a, 3);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[A:%.*]], i32 1
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_vgetq_lane_u64(uint64x2_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 1);
+#else /* POLYMORPHIC */
+    return vgetq_lane_u64(a, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vgetq_lane_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <16 x i8> [[A:%.*]], i32 1
+// CHECK-NEXT:    ret i8 [[TMP0]]
+//
+uint8_t test_vgetq_lane_u8(uint8x16_t a)
+{
+#ifdef POLYMORPHIC
+    return vgetq_lane(a, 1);
+#else /* POLYMORPHIC */
+    return vgetq_lane_u8(a, 1);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[A_COERCE:%.*]] to i32
+// CHECK-NEXT:    [[TMP_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[TMP0]] to i16
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16 [[TMP_0_EXTRACT_TRUNC]] to half
+// CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x half> [[B:%.*]], half [[TMP1]], i32 4
+// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+//
+float16x8_t test_vsetq_lane_f16(float16_t a, float16x8_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 4);
+#else /* POLYMORPHIC */
+    return vsetq_lane_f16(a, b, 4);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A:%.*]], i32 2
+// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+//
+float32x4_t test_vsetq_lane_f32(float32_t a, float32x4_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 2);
+#else /* POLYMORPHIC */
+    return vsetq_lane_f32(a, b, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <16 x i8> [[B:%.*]], i8 [[A:%.*]], i32 12
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_vsetq_lane_s8(int8_t a, int8x16_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 12);
+#else /* POLYMORPHIC */
+    return vsetq_lane_s8(a, b, 12);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i16> [[B:%.*]], i16 [[A:%.*]], i32 6
+// CHECK-NEXT:    ret <8 x i16> [[TMP0]]
+//
+int16x8_t test_vsetq_lane_s16(int16_t a, int16x8_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 6);
+#else /* POLYMORPHIC */
+    return vsetq_lane_s16(a, b, 6);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> [[B:%.*]], i32 [[A:%.*]], i32 2
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+int32x4_t test_vsetq_lane_s32(int32_t a, int32x4_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 2);
+#else /* POLYMORPHIC */
+    return vsetq_lane_s32(a, b, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> [[B:%.*]], i64 [[A:%.*]], i32 0
+// CHECK-NEXT:    ret <2 x i64> [[TMP0]]
+//
+int64x2_t test_vsetq_lane_s64(int64_t a, int64x2_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 0);
+#else /* POLYMORPHIC */
+    return vsetq_lane_s64(a, b, 0);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <16 x i8> [[B:%.*]], i8 [[A:%.*]], i32 2
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_vsetq_lane_u8(uint8_t a, uint8x16_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 2);
+#else /* POLYMORPHIC */
+    return vsetq_lane_u8(a, b, 2);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i16> [[B:%.*]], i16 [[A:%.*]], i32 7
+// CHECK-NEXT:    ret <8 x i16> [[TMP0]]
+//
+uint16x8_t test_vsetq_lane_u16(uint16_t a, uint16x8_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 7);
+#else /* POLYMORPHIC */
+    return vsetq_lane_u16(a, b, 7);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> [[B:%.*]], i32 [[A:%.*]], i32 0
+// CHECK-NEXT:    ret <4 x i32> [[TMP0]]
+//
+uint32x4_t test_vsetq_lane_u32(uint32_t a, uint32x4_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 0);
+#else /* POLYMORPHIC */
+    return vsetq_lane_u32(a, b, 0);
+#endif /* POLYMORPHIC */
+}
+
+// CHECK-LABEL: @test_vsetq_lane_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> [[B:%.*]], i64 [[A:%.*]], i32 1
+// CHECK-NEXT:    ret <2 x i64> [[TMP0]]
+//
+uint64x2_t test_vsetq_lane_u64(uint64_t a, uint64x2_t b)
+{
+#ifdef POLYMORPHIC
+    return vsetq_lane(a, b, 1);
+#else /* POLYMORPHIC */
+    return vsetq_lane_u64(a, b, 1);
+#endif /* POLYMORPHIC */
+}

clang/test/CodeGen/arm-mve-intrinsics/load-store.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vld1q_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast half* [[BASE:%.*]] to <8 x half>*
 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x half>, <8 x half>* [[TMP0]], align 2
 // CHECK-NEXT:    ret <8 x half> [[TMP1]]

clang/test/CodeGen/arm-mve-intrinsics/scalar-shifts.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_urshrl(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 [[VALUE:%.*]], 32
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
 // CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[VALUE]] to i32

clang/test/CodeGen/arm-mve-intrinsics/scatter-gather.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vldrbq_gather_offset_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0i8.v8i16(i8* [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 8, i32 0, i32 0)
 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
 //

clang/test/CodeGen/arm-mve-intrinsics/vadc.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vadciq_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call { <4 x i32>, i32 } @llvm.arm.mve.vadc.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP0]], 1
 // CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 29

clang/test/CodeGen/arm-mve-intrinsics/vaddq.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vaddq_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i32> [[A:%.*]], [[B:%.*]]
 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 //

clang/test/CodeGen/arm-mve-intrinsics/vcvt.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vcvttq_f16_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vcvt.narrow(<8 x half> [[A:%.*]], <4 x float> [[B:%.*]], i32 1)
 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
 //

clang/test/CodeGen/arm-mve-intrinsics/vld24.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg -sroa -early-cse | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg -sroa -early-cse | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg -sroa -early-cse | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg -sroa -early-cse | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vld2q_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = call { <8 x half>, <8 x half> } @llvm.arm.mve.vld2q.v8f16.p0f16(half* [[ADDR:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <8 x half>, <8 x half> } [[TMP0]], 0
 // CHECK-NEXT:    [[TMP2:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T:%.*]] undef, <8 x half> [[TMP1]], 0, 0

clang/test/CodeGen/arm-mve-intrinsics/vldr.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vldrwq_gather_base_wb_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* [[ADDR:%.*]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> [[TMP0]], i32 80)
 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP1]], 1

clang/test/CodeGen/arm-mve-intrinsics/vminvq.c

 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -mfloat-abi hard -fallow-half-arguments-and-returns -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 
 #include <arm_mve.h>
 
 // CHECK-LABEL: @test_vminvq_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[A:%.*]] to i32
 // CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.arm.mve.minv.s.v16i8(i32 [[TMP0]], <16 x i8> [[B:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8

clang/test/Sema/arm-mve-immediates.c

-// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -target-feature +mve.fp -verify -fsyntax-only %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -verify -fsyntax-only %s
 
 #include <arm_mve.h>
 
 void test_load_offsets(uint32x4_t addr32, uint64x2_t addr64)
 {
   // Offsets that should be a multiple of 8 times 0,1,...,127
   vldrdq_gather_base_s64(addr64, 0);
   vldrdq_gather_base_s64(addr64, 8);
void test_load_offsets(uint32x4_t addr32, uint64x2_t addr64)
   vstrwq_scatter_base(addr32, 125*4, addr32);
   vstrwq_scatter_base(addr32, 126*4, addr32);
   vstrwq_scatter_base(addr32, 127*4, addr32);
   vstrwq_scatter_base(addr32, -4, addr32); // expected-error {{argument value -4 is outside the valid range [0, 508]}}
   vstrwq_scatter_base(addr32, 128*4, addr32); // expected-error {{argument value 512 is outside the valid range [0, 508]}}
   vstrwq_scatter_base(addr32, 2, addr32); // expected-error {{argument should be a multiple of 4}}
   vstrwq_scatter_base(addr32, 1, addr32); // expected-error {{argument should be a multiple of 4}}
 }
+
+void test_lane_indices(uint8x16_t v16, uint16x8_t v8,
+                       uint32x4_t v4, uint64x2_t v2)
+{
+  vgetq_lane_u8(v16, -1); // expected-error {{argument value -1 is outside the valid range [0, 15]}}
+  vgetq_lane_u8(v16, 0);
+  vgetq_lane_u8(v16, 15);
+  vgetq_lane_u8(v16, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}}
+
+  vgetq_lane_u16(v8, -1); // expected-error {{argument value -1 is outside the valid range [0, 7]}}
+  vgetq_lane_u16(v8, 0);
+  vgetq_lane_u16(v8, 7);
+  vgetq_lane_u16(v8, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+
+  vgetq_lane_u32(v4, -1); // expected-error {{argument value -1 is outside the valid range [0, 3]}}
+  vgetq_lane_u32(v4, 0);
+  vgetq_lane_u32(v4, 3);
+  vgetq_lane_u32(v4, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+
+  vgetq_lane_u64(v2, -1); // expected-error {{argument value -1 is outside the valid range [0, 1]}}
+  vgetq_lane_u64(v2, 0);
+  vgetq_lane_u64(v2, 1);
+  vgetq_lane_u64(v2, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}}
+
+  vsetq_lane_u8(23, v16, -1); // expected-error {{argument value -1 is outside the valid range [0, 15]}}
+  vsetq_lane_u8(23, v16, 0);
+  vsetq_lane_u8(23, v16, 15);
+  vsetq_lane_u8(23, v16, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}}
+
+  vsetq_lane_u16(23, v8, -1); // expected-error {{argument value -1 is outside the valid range [0, 7]}}
+  vsetq_lane_u16(23, v8, 0);
+  vsetq_lane_u16(23, v8, 7);
+  vsetq_lane_u16(23, v8, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+
+  vsetq_lane_u32(23, v4, -1); // expected-error {{argument value -1 is outside the valid range [0, 3]}}
+  vsetq_lane_u32(23, v4, 0);
+  vsetq_lane_u32(23, v4, 3);
+  vsetq_lane_u32(23, v4, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+
+  vsetq_lane_u64(23, v2, -1); // expected-error {{argument value -1 is outside the valid range [0, 1]}}
+  vsetq_lane_u64(23, v2, 0);
+  vsetq_lane_u64(23, v2, 1);
+  vsetq_lane_u64(23, v2, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}}
+}

clang/utils/TableGen/MveEmitter.cpp

public:
 
   // Functions that translate the Tablegen representation of an intrinsic's
   // code generation into a collection of Value objects (which will then be
   // reprocessed to read out the actual C++ code included by CGBuiltin.cpp).
   Result::Ptr getCodeForDag(DagInit *D, const Result::Scope &Scope,
                             const Type *Param);
   Result::Ptr getCodeForDagArg(DagInit *D, unsigned ArgNum,
                                const Result::Scope &Scope, const Type *Param);
-  Result::Ptr getCodeForArg(unsigned ArgNum, const Type *ArgType);
+  Result::Ptr getCodeForArg(unsigned ArgNum, const Type *ArgType,
+                            bool Promote);
 
   // Constructor and top-level functions.
 
   MveEmitter(RecordKeeper &Records);
 
   void EmitHeader(raw_ostream &OS);
   void EmitBuiltinDef(raw_ostream &OS);
   void EmitBuiltinSema(raw_ostream &OS);
   void EmitBuiltinCG(raw_ostream &OS);
   void EmitBuiltinAliases(raw_ostream &OS);
 };
 
 const Type *MveEmitter::getType(Init *I, const Type *Param) {
   if (auto Dag = dyn_cast<DagInit>(I))
     return getType(Dag, Param);
   if (auto Def = dyn_cast<DefInit>(I))
     return getType(Def->getDef(), Param);
 
   PrintFatalError("Could not convert this value into a type");
 }
 
 const Type *MveEmitter::getType(Record *R, const Type *Param) {
+  // Pass to a subfield of any wrapper records. We don't expect more than one
+  // of these: immediate operands are used as plain numbers rather than as
+  // llvm::Value, so it's meaningless to promote their type anyway.
   if (R->isSubClassOf("Immediate"))
-    R = R->getValueAsDef("type"); // pass to subfield
+    R = R->getValueAsDef("type");
+  else if (R->isSubClassOf("unpromoted"))
+    R = R->getValueAsDef("underlying_type");
 
   if (R->getName() == "Void")
     return getVoidType();
   if (R->isSubClassOf("PrimitiveType"))
     return getScalarType(R);
   if (R->isSubClassOf("ComplexType"))
     return getType(R->getValueAsDag("spec"), Param);
 
if (Rec->isSubClassOf("Type")) {
       const Type *T = getType(Rec, Param);
       return std::make_shared<TypeResult>(T);
     }
   }
 
   PrintFatalError("bad dag argument type for code generation");
 }
 
-Result::Ptr MveEmitter::getCodeForArg(unsigned ArgNum, const Type *ArgType) {
+Result::Ptr MveEmitter::getCodeForArg(unsigned ArgNum, const Type *ArgType,
+                                      bool Promote) {
   Result::Ptr V =
       std::make_shared<BuiltinArgResult>(ArgNum, isa<PointerType>(ArgType));
 
   if (const auto *ST = dyn_cast<ScalarType>(ArgType)) {
-    if (ST->isInteger() && ST->sizeInBits() < 32)
+    if (Promote && ST->isInteger() && ST->sizeInBits() < 32)
       V = std::make_shared<IntCastResult>(getScalarType("u32"), V);
   } else if (const auto *PT = dyn_cast<PredicateType>(ArgType)) {
     V = std::make_shared<IntCastResult>(getScalarType("u32"), V);
     V = std::make_shared<IRIntrinsicResult>("arm_mve_pred_i2v",
                                             std::vector<const Type *>{PT},
                                             std::vector<Result::Ptr>{V});
   }
 
ACLEIntrinsic::ACLEIntrinsic(MveEmitter &ME, Record *R, const Type *Param)
   NonEvaluating = R->getValueAsBit("nonEvaluating");
 
   // Process the intrinsic's argument list.
   DagInit *ArgsDag = R->getValueAsDag("args");
   Result::Scope Scope;
   for (unsigned i = 0, e = ArgsDag->getNumArgs(); i < e; ++i) {
     Init *TypeInit = ArgsDag->getArg(i);
 
+    bool Promote = true;
+    if (auto TypeDI = dyn_cast<DefInit>(TypeInit))
+      if (TypeDI->getDef()->isSubClassOf("unpromoted"))
+        Promote = false;
+
     // Work out the type of the argument, for use in the function prototype in
     // the header file.
     const Type *ArgType = ME.getType(TypeInit, Param);
     ArgTypes.push_back(ArgType);
 
     // The argument will usually have a name in the arguments dag, which goes
     // into the variable-name scope that the code gen will refer to.
     StringRef ArgName = ArgsDag->getArgNameStr(i);
     if (!ArgName.empty())
-      Scope[ArgName] = ME.getCodeForArg(i, ArgType);
+      Scope[ArgName] = ME.getCodeForArg(i, ArgType, Promote);
 
     // If the argument is a subclass of Immediate, record the details about
     // what values it can take, for Sema checking.
     if (auto TypeDI = dyn_cast<DefInit>(TypeInit)) {
       Record *TypeRec = TypeDI->getDef();
       if (TypeRec->isSubClassOf("Immediate")) {
         Record *Bounds = TypeRec->getValueAsDef("bounds");
         ImmediateArg &IA = ImmediateArgs[i];
         if (Bounds->isSubClassOf("IB_ConstRange")) {
           IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
           IA.i1 = Bounds->getValueAsInt("lo");
           IA.i2 = Bounds->getValueAsInt("hi");
         } else if (Bounds->getName() == "IB_UEltValue") {
           IA.boundsType = ImmediateArg::BoundsType::UInt;
           IA.i1 = Param->sizeInBits();
         } else if (Bounds->getName() == "IB_LaneIndex") {
           IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
           IA.i1 = 0;
-          IA.i2 = 128 / Param->sizeInBits();
+          IA.i2 = 128 / Param->sizeInBits() - 1;
         } else if (Bounds->getName() == "IB_EltBit") {
           IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
           IA.i1 = Bounds->getValueAsInt("base");
           IA.i2 = IA.i1 + Param->sizeInBits() - 1;
         } else {
           PrintFatalError("unrecognised ImmediateBounds subclass");
         }