ARM v8.1a adds Advanced SIMD instructions for Rounding Double Multiply Add/Subtract.

[labrinea]

Summary

The following instructions are added to AArch32 instruction set:

• VQRDMLAH: Vector Saturating Rounding Doubling Multiply Accumulate Returning High Half
• VQRDMLSH: Vector Saturating Rounding Doubling Multiply Subtract Returning High Half

The following instructions are added to AArch64 instruction set:

• SQRDMLAH: Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
• SQRDMLSH: Signed Saturating Rounding Doubling Multiply Subtract Returning High Half

This patch adds intrinsic and ACLE macro support for these instructions, as well as corresponding tests.

[t.p.northover]

Do these get the right diagnostics when used on CPUs without the new feature? I can't see how __ARM_FEATURE_QRDMX gets wired through to arm_neon.h.

[labrinea]

@t.p.northover you were right, my patch was missing predefined guard macros for the instrinsics. I've now updated the patch.

[t.p.northover]

Thanks, LGTM!

Tim.

[pengbins]

cfe/trunk/include/clang/Basic/arm_neon.td

377 ↗

(On Diff #41344)

@labrinea
It seems QRDMLSH(p0, p1, p2) is not equivelent to with vqsub( p0, vqrdmulh(p1, p2)).

• QRDMLSH(p0, p1, p2)

accum = ((p0 << esize) - 2 * (p1 * p2) + rounding_const);
ret = SignedSatQ(accum >> esize, esize);

p0<< esize + rounding_const

• vqsub( p0, vqrdmulh(p1, p2))

temp = SignedSatQ( (2 * (p1 * p2) + rounding_const) >> esize );
ret = SignedSat( p0  -  temp);

p0<< esize - rounding_const

Here is an example where the results are not same.
vqrdmlshq_s16 ( -197, -512, 11040) = -24
vqsubq_s16( -197, vqrdmulhq_n_s16(-512, 11040)) = -25

include/clang/Basic/arm_neon.td

 def OP_QDMLAL_LN : Op<(call "vqdmlal", $p0, $p1, (splat $p2, $p3))>;
 def OP_QDMLALHi_LN : Op<(call "vqdmlal", $p0, (call "vget_high", $p1),
                                               (splat $p2, $p3))>;
 def OP_QDMLSL_LN : Op<(call "vqdmlsl", $p0, $p1, (splat $p2, $p3))>;
 def OP_QDMLSLHi_LN : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1),
                                               (splat $p2, $p3))>;
 def OP_QDMULH_LN : Op<(call "vqdmulh", $p0, (splat $p1, $p2))>;
 def OP_QRDMULH_LN : Op<(call "vqrdmulh", $p0, (splat $p1, $p2))>;
+def OP_QRDMLAH : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, $p2))>;
+def OP_QRDMLSH : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, $p2))>;
+def OP_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, (splat $p2, $p3)))>;
+def OP_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (splat $p2, $p3)))>;
 def OP_FMS_LN   : Op<(call "vfma_lane", $p0, $p1, (op "-", $p2), $p3)>;
 def OP_FMS_LNQ  : Op<(call "vfma_laneq", $p0, $p1, (op "-", $p2), $p3)>;
 def OP_TRN1     : Op<(shuffle $p0, $p1, (interleave (decimate mask0, 2),
                                                     (decimate mask1, 2)))>;
 def OP_ZIP1     : Op<(shuffle $p0, $p1, (lowhalf (interleave mask0, mask1)))>;
 def OP_UZP1     : Op<(shuffle $p0, $p1, (add (decimate mask0, 2),
                                              (decimate mask1, 2)))>;
 def OP_TRN2     : Op<(shuffle $p0, $p1, (interleave
def OP_SCALAR_VMULX_LNQ : LOp<[(save_temp $x, (call "vget_lane", $p0,
                                                                      "0"))]>;
 class ScalarMulOp<string opname> :
   Op<(call opname, $p0, (call "vget_lane", $p1, $p2))>;
 
 def OP_SCALAR_QDMULL_LN : ScalarMulOp<"vqdmull">;
 def OP_SCALAR_QDMULH_LN : ScalarMulOp<"vqdmulh">;
 def OP_SCALAR_QRDMULH_LN : ScalarMulOp<"vqrdmulh">;
 
+def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1,
+                              (call "vget_lane", $p2, $p3)))>;
+def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1,
+                              (call "vget_lane", $p2, $p3)))>;
+
 def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t",
                                    (call "vget_lane",
                                          (bitcast "int16x4_t", $p0), $p1))>;
 def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t",
                                     (call "vget_lane",
                                           (bitcast "int16x8_t", $p0), $p1))>;
 def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t",
                                    (call "vset_lane",

 def VMUL     : IOpInst<"vmul", "ddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MUL>;
 def VMULP    : SInst<"vmul", "ddd", "PcQPc">;
 def VMLA     : IOpInst<"vmla", "dddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLA>;
 def VMLAL    : SOpInst<"vmlal", "wwdd", "csiUcUsUi", OP_MLAL>;
 def VMLS     : IOpInst<"vmls", "dddd", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLS>;
 def VMLSL    : SOpInst<"vmlsl", "wwdd", "csiUcUsUi", OP_MLSL>;
 def VQDMULH  : SInst<"vqdmulh", "ddd", "siQsQi">;
 def VQRDMULH : SInst<"vqrdmulh", "ddd", "siQsQi">;
+def VQRDMLAH : SOpInst<"vqrdmlah", "dddd", "siQsQi", OP_QRDMLAH>;
+def VQRDMLSH : SOpInst<"vqrdmlsh", "dddd", "siQsQi", OP_QRDMLSH>;
 def VQDMLAL  : SInst<"vqdmlal", "wwdd", "si">;
 def VQDMLSL  : SInst<"vqdmlsl", "wwdd", "si">;
 def VMULL    : SInst<"vmull", "wdd", "csiUcUsUiPc">;
 def VQDMULL  : SInst<"vqdmull", "wdd", "si">;
 
 ////////////////////////////////////////////////////////////////////////////////
 // E.3.3 Subtraction
 def VSUB    : IOpInst<"vsub", "ddd",

 def VMULL_N       : SInst<"vmull_n", "wda", "siUsUi">;
 def VMULL_LANE    : SOpInst<"vmull_lane", "wddi", "siUsUi", OP_MULL_LN>;
 def VQDMULL_N     : SInst<"vqdmull_n", "wda", "si">;
 def VQDMULL_LANE  : SOpInst<"vqdmull_lane", "wddi", "si", OP_QDMULL_LN>;
 def VQDMULH_N     : SInst<"vqdmulh_n", "dda", "siQsQi">;
 def VQDMULH_LANE  : SOpInst<"vqdmulh_lane", "ddgi", "siQsQi", OP_QDMULH_LN>;
 def VQRDMULH_N    : SInst<"vqrdmulh_n", "dda", "siQsQi">;
 def VQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "ddgi", "siQsQi", OP_QRDMULH_LN>;
+def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "dddgi", "siQsQi", OP_QRDMLAH_LN>;
+def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "dddgi", "siQsQi", OP_QRDMLSH_LN>;
 def VMLA_N        : IOpInst<"vmla_n", "ddda", "siUsUifQsQiQUsQUiQf", OP_MLA_N>;
 def VMLAL_N       : SOpInst<"vmlal_n", "wwda", "siUsUi", OP_MLAL_N>;
 def VQDMLAL_N     : SInst<"vqdmlal_n", "wwda", "si">;
 def VMLS_N        : IOpInst<"vmls_n", "ddds", "siUsUifQsQiQUsQUiQf", OP_MLS_N>;
 def VMLSL_N       : SOpInst<"vmlsl_n", "wwda", "siUsUi", OP_MLSL_N>;
 def VQDMLSL_N     : SInst<"vqdmlsl_n", "wwda", "si">;
 
 ////////////////////////////////////////////////////////////////////////////////

 def VQDMULL_LANEQ  : SOpInst<"vqdmull_laneq", "wdki", "si", OP_QDMULL_LN>;
 def VQDMULL_HIGH_LANE   : SOpInst<"vqdmull_high_lane", "wkdi", "si",
                                   OP_QDMULLHi_LN>;
 def VQDMULL_HIGH_LANEQ  : SOpInst<"vqdmull_high_laneq", "wkki", "si",
                                   OP_QDMULLHi_LN>;
 
 def VQDMULH_LANEQ  : SOpInst<"vqdmulh_laneq", "ddji", "siQsQi", OP_QDMULH_LN>;
 def VQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "ddji", "siQsQi", OP_QRDMULH_LN>;
+def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "dddji", "siQsQi", OP_QRDMLAH_LN>;
+def VQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "dddji", "siQsQi", OP_QRDMLSH_LN>;
 
 // Note: d type implemented by SCALAR_VMULX_LANE
 def VMULX_LANE : IOpInst<"vmulx_lane", "ddgi", "fQfQd", OP_MULX_LN>;
 // Note: d type is implemented by SCALAR_VMULX_LANEQ
 def VMULX_LANEQ : IOpInst<"vmulx_laneq", "ddji", "fQfQd", OP_MULX_LN>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Across vectors class

 // Scalar Integer Saturating Doubling Multiply Half High
 def SCALAR_SQDMULH : SInst<"vqdmulh", "sss", "SsSi">;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Scalar Integer Saturating Rounding Doubling Multiply Half High
 def SCALAR_SQRDMULH : SInst<"vqrdmulh", "sss", "SsSi">;
 
 ////////////////////////////////////////////////////////////////////////////////
+// Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
+def SCALAR_SQRDMLAH : SOpInst<"vqrdmlah", "ssss", "SsSi", OP_QRDMLAH>;
+
+////////////////////////////////////////////////////////////////////////////////
+// Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
+def SCALAR_SQRDMLSH : SOpInst<"vqrdmlsh", "ssss", "SsSi", OP_QRDMLSH>;
+
+////////////////////////////////////////////////////////////////////////////////
 // Scalar Floating-point Multiply Extended
 def SCALAR_FMULX : IInst<"vmulx", "sss", "SfSd">;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Scalar Floating-point Reciprocal Step
 def SCALAR_FRECPS : IInst<"vrecps", "sss", "SfSd">;
 
 ////////////////////////////////////////////////////////////////////////////////

 // Scalar Integer Saturating Doubling Multiply Half High (scalar by element)
 def SCALAR_SQDMULH_LANE : SOpInst<"vqdmulh_lane", "ssdi", "SsSi", OP_SCALAR_QDMULH_LN>;
 def SCALAR_SQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "ssji", "SsSi", OP_SCALAR_QDMULH_LN>;
 
 // Scalar Integer Saturating Rounding Doubling Multiply Half High
 def SCALAR_SQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "ssdi", "SsSi", OP_SCALAR_QRDMULH_LN>;
 def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "ssji", "SsSi", OP_SCALAR_QRDMULH_LN>;
 
+// Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half
+def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "sssdi", "SsSi", OP_SCALAR_QRDMLAH_LN>;
+def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "sssji", "SsSi", OP_SCALAR_QRDMLAH_LN>;
+
+// Signed Saturating Rounding Doubling Multiply Subtract Returning High Half
+def SCALAR_SQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "sssdi", "SsSi", OP_SCALAR_QRDMLSH_LN>;
+def SCALAR_SQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "sssji", "SsSi", OP_SCALAR_QRDMLSH_LN>;
+
 def SCALAR_VDUP_LANE : IInst<"vdup_lane", "sdi", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
 def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "sji", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">;
 }

lib/Basic/Targets.cpp

void getTargetDefines(const LangOptions &Opts,
     }
 
     // ACLE 6.4.6 Q (saturation) flag
     if (DSP || SAT)
       Builder.defineMacro("__ARM_FEATURE_QBIT", "1");
 
     if (Opts.UnsafeFPMath)
       Builder.defineMacro("__ARM_FP_FAST", "1");
+
+    if (ArchKind == llvm::ARM::AK_ARMV8_1A)
+      Builder.defineMacro("__ARM_FEATURE_QRDMX", "1");
   }
 
   ArrayRef<Builtin::Info> getTargetBuiltins() const override {
     return llvm::makeArrayRef(BuiltinInfo,
                              clang::ARM::LastTSBuiltin-Builtin::FirstTSBuiltin);
   }
   bool isCLZForZeroUndef() const override { return false; }
   BuiltinVaListKind getBuiltinVaListKind() const override {
enum FPUModeEnum {
     FPUMode,
     NeonMode
   };
 
   unsigned FPU;
   unsigned CRC;
   unsigned Crypto;
   unsigned Unaligned;
+  unsigned V8_1A;
 
   static const Builtin::Info BuiltinInfo[];
 
   std::string ABI;
 
 public:
   AArch64TargetInfo(const llvm::Triple &Triple)
       : TargetInfo(Triple), ABI("aapcs") {
if (CRC)
       Builder.defineMacro("__ARM_FEATURE_CRC32", "1");
 
     if (Crypto)
       Builder.defineMacro("__ARM_FEATURE_CRYPTO", "1");
 
     if (Unaligned)
       Builder.defineMacro("__ARM_FEATURE_UNALIGNED", "1");
 
+    if (V8_1A)
+      Builder.defineMacro("__ARM_FEATURE_QRDMX", "1");
+
     // All of the __sync_(bool|val)_compare_and_swap_(1|2|4|8) builtins work.
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4");
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8");
   }
 
   ArrayRef<Builtin::Info> getTargetBuiltins() const override {
public:
   }
 
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override {
     FPU = FPUMode;
     CRC = 0;
     Crypto = 0;
     Unaligned = 1;
+    V8_1A = 0;
 
     for (const auto &Feature : Features) {
       if (Feature == "+neon")
         FPU = NeonMode;
       if (Feature == "+crc")
         CRC = 1;
       if (Feature == "+crypto")
         Crypto = 1;
       if (Feature == "+strict-align")
         Unaligned = 0;
+      if (Feature == "+v8.1a")
+        V8_1A = 1;
     }
 
     setDataLayoutString();
 
     return true;
   }
 
   bool isCLZForZeroUndef() const override { return false; }

test/CodeGen/aarch64-neon-2velem.c

 }
 
 int32x4_t test_vqrdmulhq_lane_s32(int32x4_t a, int32x2_t v) {
   // CHECK-LABEL: test_vqrdmulhq_lane_s32
   return vqrdmulhq_lane_s32(a, v, 1);
   // CHECK: sqrdmulh {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[1]
 }
 
+int16x4_t test_vqrdmlah_lane_s16(int16x4_t a, int16x4_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlah_lane_s16
+  return vqrdmlah_lane_s16(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[3]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlahq_lane_s16(int16x8_t a, int16x8_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_lane_s16
+  return vqrdmlahq_lane_s16(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[3]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlah_lane_s32(int32x2_t a, int32x2_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlah_lane_s32
+  return vqrdmlah_lane_s32(a, b, v, 1);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[1]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlahq_lane_s32(int32x4_t a, int32x4_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_lane_s32
+  return vqrdmlahq_lane_s32(a, b, v, 1);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[1]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
+int16x4_t test_vqrdmlsh_lane_s16(int16x4_t a, int16x4_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_lane_s16
+  return vqrdmlsh_lane_s16(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[3]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlshq_lane_s16(int16x8_t a, int16x8_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_lane_s16
+  return vqrdmlshq_lane_s16(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[3]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlsh_lane_s32(int32x2_t a, int32x2_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_lane_s32
+  return vqrdmlsh_lane_s32(a, b, v, 1);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[1]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlshq_lane_s32(int32x4_t a, int32x4_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_lane_s32
+  return vqrdmlshq_lane_s32(a, b, v, 1);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[1]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
 float32x2_t test_vmul_lane_f32(float32x2_t a, float32x2_t v) {
   // CHECK-LABEL: test_vmul_lane_f32
   return vmul_lane_f32(a, v, 1);
   // CHECK: fmul {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[1]
 }
 
 
 float64x1_t test_vmul_lane_f64(float64x1_t a, float64x1_t v) {

 }
 
 int32x4_t test_vqrdmulhq_lane_s32_0(int32x4_t a, int32x2_t v) {
   // CHECK-LABEL: test_vqrdmulhq_lane_s32_0
   return vqrdmulhq_lane_s32(a, v, 0);
   // CHECK: sqrdmulh {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
 }
 
+int16x4_t test_vqrdmlah_lane_s16_0(int16x4_t a, int16x4_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlah_lane_s16_0
+  return vqrdmlah_lane_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlahq_lane_s16_0(int16x8_t a, int16x8_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_lane_s16_0
+  return vqrdmlahq_lane_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlah_lane_s32_0(int32x2_t a, int32x2_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlah_lane_s32_0
+  return vqrdmlah_lane_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlahq_lane_s32_0(int32x4_t a, int32x4_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_lane_s32_0
+  return vqrdmlahq_lane_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
+int16x4_t test_vqrdmlsh_lane_s16_0(int16x4_t a, int16x4_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_lane_s16_0
+  return vqrdmlsh_lane_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlshq_lane_s16_0(int16x8_t a, int16x8_t b, int16x4_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_lane_s16_0
+  return vqrdmlshq_lane_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlsh_lane_s32_0(int32x2_t a, int32x2_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_lane_s32_0
+  return vqrdmlsh_lane_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlshq_lane_s32_0(int32x4_t a, int32x4_t b, int32x2_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_lane_s32_0
+  return vqrdmlshq_lane_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
 float32x2_t test_vmul_lane_f32_0(float32x2_t a, float32x2_t v) {
   // CHECK-LABEL: test_vmul_lane_f32_0
   return vmul_lane_f32(a, v, 0);
   // CHECK: fmul {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
 }
 
 float32x4_t test_vmulq_lane_f32_0(float32x4_t a, float32x2_t v) {
   // CHECK-LABEL: test_vmulq_lane_f32_0

 }
 
 int32x4_t test_vqrdmulhq_laneq_s32_0(int32x4_t a, int32x4_t v) {
   // CHECK-LABEL: test_vqrdmulhq_laneq_s32_0
   return vqrdmulhq_laneq_s32(a, v, 0);
   // CHECK: sqrdmulh {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
 }
 
+int16x4_t test_vqrdmlah_laneq_s16_0(int16x4_t a, int16x4_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlah_laneq_s16_0
+  return vqrdmlah_laneq_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlahq_laneq_s16_0(int16x8_t a, int16x8_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_laneq_s16_0
+  return vqrdmlahq_laneq_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlah_laneq_s32_0(int32x2_t a, int32x2_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlah_laneq_s32_0
+  return vqrdmlah_laneq_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlahq_laneq_s32_0(int32x4_t a, int32x4_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_laneq_s32_0
+  return vqrdmlahq_laneq_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
+int16x4_t test_vqrdmlsh_laneq_s16_0(int16x4_t a, int16x4_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_laneq_s16_0
+  return vqrdmlsh_laneq_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlshq_laneq_s16_0(int16x8_t a, int16x8_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_laneq_s16_0
+  return vqrdmlshq_laneq_s16(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlsh_laneq_s32_0(int32x2_t a, int32x2_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_laneq_s32_0
+  return vqrdmlsh_laneq_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlshq_laneq_s32_0(int32x4_t a, int32x4_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_laneq_s32_0
+  return vqrdmlshq_laneq_s32(a, b, v, 0);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[0]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
 uint16x4_t test_vmla_lane_u16(uint16x4_t a, uint16x4_t b, uint16x4_t v) {
   // CHECK-LABEL: test_vmla_lane_u16
   return vmla_lane_u16(a, b, v, 3);
   // CHECK: mla {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.h[3]
 }
 
 uint16x8_t test_vmlaq_lane_u16(uint16x8_t a, uint16x8_t b, uint16x4_t v) {
   // CHECK-LABEL: test_vmlaq_lane_u16

 }
 
 int32x4_t test_vqrdmulhq_laneq_s32(int32x4_t a, int32x4_t v) {
   // CHECK-LABEL: test_vqrdmulhq_laneq_s32
   return vqrdmulhq_laneq_s32(a, v, 3);
   // CHECK: sqrdmulh {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.s[3]
 }
 
+int16x4_t test_vqrdmlah_laneq_s16(int16x4_t a, int16x4_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlah_laneq_s16
+  return vqrdmlah_laneq_s16(a, b, v, 7);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[7]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlahq_laneq_s16(int16x8_t a, int16x8_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_laneq_s16
+  return vqrdmlahq_laneq_s16(a, b, v, 7);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[7]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlah_laneq_s32(int32x2_t a, int32x2_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlah_laneq_s32
+  return vqrdmlah_laneq_s32(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[3]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlahq_laneq_s32(int32x4_t a, int32x4_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlahq_laneq_s32
+  return vqrdmlahq_laneq_s32(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[3]
+  // CHECK-NEXT: sqadd {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
+int16x4_t test_vqrdmlsh_laneq_s16(int16x4_t a, int16x4_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_laneq_s16
+  return vqrdmlsh_laneq_s16(a, b, v, 7);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.h[7]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int16x8_t test_vqrdmlshq_laneq_s16(int16x8_t a, int16x8_t b, int16x8_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_laneq_s16
+  return vqrdmlshq_laneq_s16(a, b, v, 7);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.h[7]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x2_t test_vqrdmlsh_laneq_s32(int32x2_t a, int32x2_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlsh_laneq_s32
+  return vqrdmlsh_laneq_s32(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.s[3]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int32x4_t test_vqrdmlshq_laneq_s32(int32x4_t a, int32x4_t b, int32x4_t v) {
+  // CHECK-LABEL: test_vqrdmlshq_laneq_s32
+  return vqrdmlshq_laneq_s32(a, b, v, 3);
+  // CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.s[3]
+  // CHECK-NEXT: sqsub {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+

test/CodeGen/aarch64-neon-intrinsics.c

 }
 
 int32x4_t test_vqrdmulhq_s32(int32x4_t a, int32x4_t b) {
 // CHECK-LABEL: test_vqrdmulhq_s32
   return vqrdmulhq_s32(a, b);
 // CHECK: sqrdmulh {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
 }
 
+int16x4_t test_vqrdmlah_s16(int16x4_t a, int16x4_t b, int16x4_t c) {
+// CHECK-LABEL: test_vqrdmlah_s16
+  return vqrdmlah_s16(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+// CHECK-NEXT: sqadd {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int32x2_t test_vqrdmlah_s32(int32x2_t a, int32x2_t b, int32x2_t c) {
+// CHECK-LABEL: test_vqrdmlah_s32
+  return vqrdmlah_s32(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
+// CHECK-NEXT: sqadd {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int16x8_t test_vqrdmlahq_s16(int16x8_t a, int16x8_t b, int16x8_t c) {
+// CHECK-LABEL: test_vqrdmlahq_s16
+  return vqrdmlahq_s16(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+// CHECK-NEXT: sqadd {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x4_t test_vqrdmlahq_s32(int32x4_t a, int32x4_t b, int32x4_t c) {
+// CHECK-LABEL: test_vqrdmlahq_s32
+  return vqrdmlahq_s32(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+// CHECK-NEXT: sqadd {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
+int16x4_t test_vqrdmlsh_s16(int16x4_t a, int16x4_t b, int16x4_t c) {
+// CHECK-LABEL: test_vqrdmlsh_s16
+  return vqrdmlsh_s16(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.4h]], {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+// CHECK-NEXT: sqsub {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, [[REG]]
+}
+
+int32x2_t test_vqrdmlsh_s32(int32x2_t a, int32x2_t b, int32x2_t c) {
+// CHECK-LABEL: test_vqrdmlsh_s32
+  return vqrdmlsh_s32(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.2s]], {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
+// CHECK-NEXT: sqsub {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, [[REG]]
+}
+
+int16x8_t test_vqrdmlshq_s16(int16x8_t a, int16x8_t b, int16x8_t c) {
+// CHECK-LABEL: test_vqrdmlshq_s16
+  return vqrdmlshq_s16(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.8h]], {{v[0-9]+}}.8h, {{v[0-9]+}}.8h
+// CHECK-NEXT: sqsub {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, [[REG]]
+}
+
+int32x4_t test_vqrdmlshq_s32(int32x4_t a, int32x4_t b, int32x4_t c) {
+// CHECK-LABEL: test_vqrdmlshq_s32
+  return vqrdmlshq_s32(a, b, c);
+// CHECK:      sqrdmulh [[REG:v[0-9]+.4s]], {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+// CHECK-NEXT: sqsub {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, [[REG]]
+}
+
 float32x2_t test_vmulx_f32(float32x2_t a, float32x2_t b) {
 // CHECK-LABEL: test_vmulx_f32
   return vmulx_f32(a, b);
 // CHECK: fmulx {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
 }
 
 float32x4_t test_vmulxq_f32(float32x4_t a, float32x4_t b) {
 // CHECK-LABEL: test_vmulxq_f32

 }
 
 int32_t test_vqrdmulhs_s32(int32_t a, int32_t b) {
 // CHECK-LABEL: test_vqrdmulhs_s32
   return vqrdmulhs_s32(a, b);
 // CHECK: sqrdmulh {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 }
 
+int16_t test_vqrdmlahh_s16(int16_t a, int16_t b, int16_t c) {
+// CHECK-LABEL: test_vqrdmlahh_s16
+  return vqrdmlahh_s16(a, b, c);
+// CHECK: sqrdmulh [[REG:h[0-9]+|v[0-9]+.4h]], {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}
+// CHECK: sqadd {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, [[REG]]
+}
+
+int32_t test_vqrdmlahs_s32(int32_t a, int32_t b, int32_t c) {
+// CHECK-LABEL: test_vqrdmlahs_s32
+  return vqrdmlahs_s32(a, b, c);
+// CHECK: sqrdmulh [[REG:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+// CHECK: sqadd {{s[0-9]+}}, {{s[0-9]+}}, [[REG]]
+}
+
+int16_t test_vqrdmlshh_s16(int16_t a, int16_t b, int16_t c) {
+// CHECK-LABEL: test_vqrdmlshh_s16
+  return vqrdmlshh_s16(a, b, c);
+// CHECK: sqrdmulh [[REG:h[0-9]+|v[0-9]+.4h]], {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}
+// CHECK: sqsub {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, [[REG]]
+}
+
+int32_t test_vqrdmlshs_s32(int32_t a, int32_t b, int32_t c) {
+// CHECK-LABEL: test_vqrdmlshs_s32
+  return vqrdmlshs_s32(a, b, c);
+// CHECK: sqrdmulh [[REG:s[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
+// CHECK: sqsub {{s[0-9]+}}, {{s[0-9]+}}, [[REG]]
+}
+
 float32_t test_vmulxs_f32(float32_t a, float32_t b) {
 // CHECK-LABEL: test_vmulxs_f32
   return vmulxs_f32(a, b);
 // CHECK: fmulx {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 }
 
 float64_t test_vmulxd_f64(float64_t a, float64_t b) {
 // CHECK-LABEL: test_vmulxd_f64

test/CodeGen/aarch64-neon-scalar-x-indexed-elem.c

 }
 
 // CHECK-LABEL: test_vqrdmulhs_lane_s32
 int32_t test_vqrdmulhs_lane_s32(int32_t a, int32x2_t b) {
   return vqrdmulhs_lane_s32(a, b, 1);
 // CHECK: sqrdmulh {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[1]
 }
 
+// CHECK-LABEL: test_vqrdmlahh_lane_s16
+int16_t test_vqrdmlahh_lane_s16(int16_t a, int16_t b, int16x4_t c) {
+  return vqrdmlahh_lane_s16(a, b, c, 3);
+// CHECK: sqrdmulh [[REG:h[0-9]+|v[0-9]+.4h]], {{h[0-9]+|v[0-9]+.4h}}, {{v[0-9]+}}.h[3]
+// CHECK: sqadd {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, [[REG]]
+}
+
+// CHECK-LABEL: test_vqrdmlahs_lane_s32
+int32_t test_vqrdmlahs_lane_s32(int32_t a, int32_t b, int32x2_t c) {
+  return vqrdmlahs_lane_s32(a, b, c, 1);
+// CHECK: sqrdmulh [[REG:s[0-9]+]], {{s[0-9]+}}, {{v[0-9]+}}.s[1]
+// CHECK: sqadd {{s[0-9]+}}, {{s[0-9]+}}, [[REG]]
+}
+
+// CHECK-LABEL: test_vqrdmlshh_lane_s16
+int16_t test_vqrdmlshh_lane_s16(int16_t a, int16_t b, int16x4_t c) {
+  return vqrdmlshh_lane_s16(a, b, c, 3);
+// CHECK: sqrdmulh [[REG:h[0-9]+|v[0-9]+.4h]], {{h[0-9]+|v[0-9]+.4h}}, {{v[0-9]+}}.h[3]
+// CHECK: sqsub {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, [[REG]]
+}
+
+// CHECK-LABEL: test_vqrdmlshs_lane_s32
+int32_t test_vqrdmlshs_lane_s32(int32_t a, int32_t b, int32x2_t c) {
+  return vqrdmlshs_lane_s32(a, b, c, 1);
+// CHECK: sqrdmulh [[REG:s[0-9]+]], {{s[0-9]+}}, {{v[0-9]+}}.s[1]
+// CHECK: sqsub {{s[0-9]+}}, {{s[0-9]+}}, [[REG]]
+}
 
 // CHECK-LABEL: test_vqrdmulhh_laneq_s16
 int16_t test_vqrdmulhh_laneq_s16(int16_t a, int16x8_t b) {
   return vqrdmulhh_laneq_s16(a, b, 7);
 // CHECK: sqrdmulh {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, {{v[0-9]+}}.h[7]
 }
 
 
 // CHECK-LABEL: test_vqrdmulhs_laneq_s32
 int32_t test_vqrdmulhs_laneq_s32(int32_t a, int32x4_t b) {
   return vqrdmulhs_laneq_s32(a, b, 3);
 // CHECK: sqrdmulh {{s[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}.s[3]
 }
 
+// CHECK-LABEL: test_vqrdmlahh_laneq_s16
+int16_t test_vqrdmlahh_laneq_s16(int16_t a, int16_t b, int16x8_t c) {
+  return vqrdmlahh_laneq_s16(a, b, c, 7);
+// CHECK: sqrdmulh [[REG:h[0-9]+|v[0-9]+.4h]], {{h[0-9]+|v[0-9]+.4h}}, {{v[0-9]+}}.h[7]
+// CHECK: sqadd {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, [[REG]]
+}
+
+// CHECK-LABEL: test_vqrdmlahs_laneq_s32
+int32_t test_vqrdmlahs_laneq_s32(int32_t a, int32_t b, int32x4_t c) {
+  return vqrdmlahs_laneq_s32(a, b, c, 3);
+// CHECK: sqrdmulh [[REG:s[0-9]+]], {{s[0-9]+}}, {{v[0-9]+}}.s[3]
+// CHECK: sqadd {{s[0-9]+}}, {{s[0-9]+}}, [[REG]]
+}
+
+// CHECK-LABEL: test_vqrdmlshh_laneq_s16
+int16_t test_vqrdmlshh_laneq_s16(int16_t a, int16_t b, int16x8_t c) {
+  return vqrdmlshh_laneq_s16(a, b, c, 7);
+// CHECK: sqrdmulh [[REG:h[0-9]+|v[0-9]+.4h]], {{h[0-9]+|v[0-9]+.4h}}, {{v[0-9]+}}.h[7]
+// CHECK: sqsub {{h[0-9]+|v[0-9]+.4h}}, {{h[0-9]+|v[0-9]+.4h}}, [[REG]]
+}
+
+// CHECK-LABEL: test_vqrdmlshs_laneq_s32
+int32_t test_vqrdmlshs_laneq_s32(int32_t a, int32_t b, int32x4_t c) {
+  return vqrdmlshs_laneq_s32(a, b, c, 3);
+// CHECK: sqrdmulh [[REG:s[0-9]+]], {{s[0-9]+}}, {{v[0-9]+}}.s[3]
+// CHECK: sqsub {{s[0-9]+}}, {{s[0-9]+}}, [[REG]]
+}
+
 // CHECK-LABEL: test_vqdmlalh_lane_s16
 int32_t test_vqdmlalh_lane_s16(int32_t a, int16_t b, int16x4_t c) {
   return vqdmlalh_lane_s16(a, b, c, 3);
 // CHECK: sqdmlal {{s[0-9]+|v[0-9]+.4s}}, {{h[0-9]+|v[0-9]+.4h}}, {{v[0-9]+}}.h[3]
 }
 
 // CHECK-LABEL: test_vqdmlals_lane_s32
 int64_t test_vqdmlals_lane_s32(int64_t a, int32_t b, int32x2_t c) {

test/CodeGen/arm_neon_intrinsics.c

 
 // CHECK-LABEL: test_vqrdmulhq_s32
 // CHECK: vqrdmulh.s32 q{{[0-9]+}}, q{{[0-9]+}}, q{{[0-9]+}}
 int32x4_t test_vqrdmulhq_s32(int32x4_t a, int32x4_t b) {
   return vqrdmulhq_s32(a, b);
 }
 
 
+// CHECK-LABEL: test_vqrdmlah_s16
+// CHECK:      vqrdmulh.s16 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}
+// CHECK-NEXT: vqadd.s16 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int16x4_t test_vqrdmlah_s16(int16x4_t a, int16x4_t b, int16x4_t c) {
+  return vqrdmlah_s16(a, b, c);
+}
+
+// CHECK-LABEL: test_vqrdmlah_s32
+// CHECK:      vqrdmulh.s32 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}
+// CHECK-NEXT: vqadd.s32 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int32x2_t test_vqrdmlah_s32(int32x2_t a, int32x2_t b, int32x2_t c) {
+  return vqrdmlah_s32(a, b, c);
+}
+
+// CHECK-LABEL: test_vqrdmlahq_s16
+// CHECK:      vqrdmulh.s16 [[REG:q[0-9]+]], q{{[0-9]+}}, q{{[0-9]+}}
+// CHECK-NEXT: vqadd.s16 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int16x8_t test_vqrdmlahq_s16(int16x8_t a, int16x8_t b, int16x8_t c) {
+  return vqrdmlahq_s16(a, b, c);
+}
+
+// CHECK-LABEL: test_vqrdmlahq_s32
+// CHECK:      vqrdmulh.s32 [[REG:q[0-9]+]], q{{[0-9]+}}, q{{[0-9]+}}
+// CHECK-NEXT: vqadd.s32 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int32x4_t test_vqrdmlahq_s32(int32x4_t a, int32x4_t b, int32x4_t c) {
+  return vqrdmlahq_s32(a, b, c);
+}
+
+
+// CHECK-LABEL: test_vqrdmlsh_s16
+// CHECK:      vqrdmulh.s16 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}
+// CHECK-NEXT: vqsub.s16 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int16x4_t test_vqrdmlsh_s16(int16x4_t a, int16x4_t b, int16x4_t c) {
+  return vqrdmlsh_s16(a, b, c);
+}
+
+// CHECK-LABEL: test_vqrdmlsh_s32
+// CHECK:      vqrdmulh.s32 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}
+// CHECK-NEXT: vqsub.s32 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int32x2_t test_vqrdmlsh_s32(int32x2_t a, int32x2_t b, int32x2_t c) {
+  return vqrdmlsh_s32(a, b, c);
+}
+
+// CHECK-LABEL: test_vqrdmlshq_s16
+// CHECK:      vqrdmulh.s16 [[REG:q[0-9]+]], q{{[0-9]+}}, q{{[0-9]+}}
+// CHECK-NEXT: vqsub.s16 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int16x8_t test_vqrdmlshq_s16(int16x8_t a, int16x8_t b, int16x8_t c) {
+  return vqrdmlshq_s16(a, b, c);
+}
+
+// CHECK-LABEL: test_vqrdmlshq_s32
+// CHECK:      vqrdmulh.s32 [[REG:q[0-9]+]], q{{[0-9]+}}, q{{[0-9]+}}
+// CHECK-NEXT: vqsub.s32 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int32x4_t test_vqrdmlshq_s32(int32x4_t a, int32x4_t b, int32x4_t c) {
+  return vqrdmlshq_s32(a, b, c);
+}
+
+
 // CHECK-LABEL: test_vqrdmulh_lane_s16
 // CHECK: vqrdmulh.s16 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
 int16x4_t test_vqrdmulh_lane_s16(int16x4_t a, int16x4_t b) {
   return vqrdmulh_lane_s16(a, b, 3);
 }
 
 // CHECK-LABEL: test_vqrdmulh_lane_s32
 // CHECK: vqrdmulh.s32 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]

 
 // CHECK-LABEL: test_vqrdmulhq_lane_s32
 // CHECK: vqrdmulh.s32 q{{[0-9]+}}, q{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
 int32x4_t test_vqrdmulhq_lane_s32(int32x4_t a, int32x2_t b) {
   return vqrdmulhq_lane_s32(a, b, 1);
 }
 
 
+// CHECK-LABEL: test_vqrdmlah_lane_s16
+// CHECK:      vqrdmulh.s16 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqadd.s16 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int16x4_t test_vqrdmlah_lane_s16(int16x4_t a, int16x4_t b, int16x4_t c) {
+  return vqrdmlah_lane_s16(a, b, c, 3);
+}
+
+// CHECK-LABEL: test_vqrdmlah_lane_s32
+// CHECK:      vqrdmulh.s32 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqadd.s32 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int32x2_t test_vqrdmlah_lane_s32(int32x2_t a, int32x2_t b, int32x2_t c) {
+  return vqrdmlah_lane_s32(a, b, c, 1);
+}
+
+// CHECK-LABEL: test_vqrdmlahq_lane_s16
+// CHECK:      vqrdmulh.s16 [[REG:q[0-9]+]], q{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqadd.s16 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int16x8_t test_vqrdmlahq_lane_s16(int16x8_t a, int16x8_t b, int16x4_t c) {
+  return vqrdmlahq_lane_s16(a, b, c, 3);
+}
+
+// CHECK-LABEL: test_vqrdmlahq_lane_s32
+// CHECK:      vqrdmulh.s32 [[REG:q[0-9]+]], q{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqadd.s32 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int32x4_t test_vqrdmlahq_lane_s32(int32x4_t a, int32x4_t b, int32x2_t c) {
+  return vqrdmlahq_lane_s32(a, b, c, 1);
+}
+
+
+// CHECK-LABEL: test_vqrdmlsh_lane_s16
+// CHECK:      vqrdmulh.s16 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqsub.s16 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int16x4_t test_vqrdmlsh_lane_s16(int16x4_t a, int16x4_t b, int16x4_t c) {
+  return vqrdmlsh_lane_s16(a, b, c, 3);
+}
+
+// CHECK-LABEL: test_vqrdmlsh_lane_s32
+// CHECK:      vqrdmulh.s32 [[REG:d[0-9]+]], d{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqsub.s32 d{{[0-9]+}}, d{{[0-9]+}}, [[REG]]
+int32x2_t test_vqrdmlsh_lane_s32(int32x2_t a, int32x2_t b, int32x2_t c) {
+  return vqrdmlsh_lane_s32(a, b, c, 1);
+}
+
+// CHECK-LABEL: test_vqrdmlshq_lane_s16
+// CHECK:      vqrdmulh.s16 [[REG:q[0-9]+]], q{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqsub.s16 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int16x8_t test_vqrdmlshq_lane_s16(int16x8_t a, int16x8_t b, int16x4_t c) {
+  return vqrdmlshq_lane_s16(a, b, c, 3);
+}
+
+// CHECK-LABEL: test_vqrdmlshq_lane_s32
+// CHECK:      vqrdmulh.s32 [[REG:q[0-9]+]], q{{[0-9]+}}, d{{[0-9]+}}[{{[0-9]}}]
+// CHECK-NEXT: vqsub.s32 q{{[0-9]+}}, q{{[0-9]+}}, [[REG]]
+int32x4_t test_vqrdmlshq_lane_s32(int32x4_t a, int32x4_t b, int32x2_t c) {
+  return vqrdmlshq_lane_s32(a, b, c, 1);
+}
+
+
 // CHECK-LABEL: test_vqrdmulh_n_s16
 // CHECK: vqrdmulh.s16 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}
 int16x4_t test_vqrdmulh_n_s16(int16x4_t a, int16_t b) {
   return vqrdmulh_n_s16(a, b);
 }
 
 // CHECK-LABEL: test_vqrdmulh_n_s32
 // CHECK: vqrdmulh.s32 d{{[0-9]+}}, d{{[0-9]+}}, d{{[0-9]+}}

test/Preprocessor/aarch64-target-features.c

 // RUN: %clang -target arm64-none-linux-gnu -fshort-enums -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SHORTENUMS %s
 // CHECK-SHORTENUMS: __ARM_SIZEOF_MINIMAL_ENUM 1
 
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+simd -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NEON %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+simd -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NEON %s
 // CHECK-NEON: __ARM_NEON 1
 // CHECK-NEON: __ARM_NEON_FP 0xE
 
+// RUN: %clang -target aarch64-none-eabi -march=armv8.1-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-V81A %s
+// CHECK-V81A: __ARM_FEATURE_QRDMX 1
+
 // RUN: %clang -target aarch64 -march=arm64 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-ARCH-NOT-ACCEPT %s
 // RUN: %clang -target aarch64 -march=aarch64 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-ARCH-NOT-ACCEPT %s
 // CHECK-ARCH-NOT-ACCEPT: error: the clang compiler does not support
 
 // RUN: %clang -target aarch64 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-GENERIC %s
 // RUN: %clang -target aarch64 -march=armv8-a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-GENERIC %s
 // CHECK-GENERIC: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon"
 

test/Preprocessor/arm-target-features.c

 // KRAIT-THUMB:#define __ARM_ARCH_EXT_IDIV__ 1
 // KRAIT-THUMB:#define __ARM_FEATURE_DSP
 // KRAIT-THUMB:#define  __ARM_VFPV4__ 1
 
 // RUN: %clang -target armv8.1a-none-none-eabi -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-V81A %s
 // CHECK-V81A: __ARM_ARCH 8
 // CHECK-V81A: __ARM_ARCH_8_1A__ 1
 // CHECK-V81A: #define __ARM_ARCH_PROFILE 'A'
+// CHECK-V81A: __ARM_FEATURE_QRDMX 1
 // CHECK-V81A: #define __ARM_FP 0xE