[RISCV] Turn splat shuffles of vector loads into scalar loads and a splat.

[craig.topper]

Summary

VectorCombine turns splats of scalar loads into a vector load and
splat if it can determine that reading extract bytes won't page
fault, isn't volatile or atomic, etc. I'm not sure how useful this
is for us. It's especially annoying because vrgather.vi has a
early clobber constraint so this can force us to use an extra
register. If there happen to be splats of neighboring scalar
loads, VectorCombine seems to create multiple vector loads starting
from the address of each scalar.

This patch reverses the transform by turning it back into a scalar
load. I'm also peeking through concat_vectors because I saw
VectorCombine pad with undefs if it was too close to the end of
an array to load the full vector size.

[frasercrmck]

How difficult would it be to opt out of this vector combine in the first place? I can imagine that this may benefit other architectures.

[craig.topper]

In D99236#2647845, @frasercrmck wrote:

How difficult would it be to opt out of this vector combine in the first place? I can imagine that this may benefit other architectures.

It looks like the VectorCombine code only considers the cost of the insert+load that it is replacing. It doesn't look at the shuffle use. So it compares the cost of vector load to the cost of a load+insert. That by itself might be profitable. But once you consider that insert+shuffle is one instruction for us, then reversing the transform makes some sense.

[craig.topper]

Ping

[HsiangKai]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1415

'the' is duplicated.

[craig.topper]

Remove duplicated "the" from comment.

[rogfer01]

One could argue that this introduces coupling between the scalar register bank and the vector register bank. But I presume the simpler scalar load makes up for that loss of decoupling.

Reassuring myself here: VectorCombine cannot do this for scalables, can it?

[craig.topper]

In D99236#2663124, @rogfer01 wrote:

One could argue that this introduces coupling between the scalar register bank and the vector register bank. But I presume the simpler scalar load makes up for that loss of decoupling.

Reassuring myself here: VectorCombine cannot do this for scalables, can it?

The VectorCombine code only works on fixed vectors

[craig.topper]

Ping.

[craig.topper]

I have a much better idea for this.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
   SDValue V2 = Op.getOperand(1);
   SDLoc DL(Op);
   MVT XLenVT = Subtarget.getXLenVT();
   MVT VT = Op.getSimpleValueType();
   unsigned NumElts = VT.getVectorNumElements();
   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
 
   if (SVN->isSplat()) {
-    int Lane = SVN->getSplatIndex();
+    const int Lane = SVN->getSplatIndex();
     if (Lane >= 0) {
+      MVT XLenVT = Subtarget.getXLenVT();
       MVT ContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
           DAG, VT, Subtarget);
+      MVT SVT = VT.getVectorElementType();
+
+      SDValue Mask, VL;
+      std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
+
+      // Turn splatted vector load into a scalar loads and splat as long as the
+      // element the scalar load is legal. VectorCombine can introduce this
+      // pattern.
+      SDValue V = V1;
+      // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
+      // with undef.
+      // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
+      int Offset = Lane;
+      if (V.getOpcode() == ISD::CONCAT_VECTORS && V.hasOneUse()) {
+        int OpElements =
+            V.getOperand(0).getSimpleValueType().getVectorNumElements();
+        V = V.getOperand(Offset / OpElements);
+        Offset %= OpElements;
+      }
+
+      // We need to ensure the load isn't atomic or volatile. For integers the
+      // scalar can't be larger than XLen. We also avoid creating an unaligned
+      // scalar load.
+      if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple() &&
+          V.hasOneUse() && (SVT.isFloatingPoint() || SVT.bitsLE(XLenVT)) &&
+          cast<LoadSDNode>(V)->getAlign() >= SVT.getStoreSize()) {
+        auto *Ld = cast<LoadSDNode>(V);
+        Offset *= SVT.getStoreSize();
+        SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
+                                                   TypeSize::Fixed(Offset), DL);
+        if (SVT.isFloatingPoint())
+          V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
+                          Ld->getPointerInfo().getWithOffset(Offset),
+                          Ld->getOriginalAlign(),
+                          Ld->getMemOperand()->getFlags());
+        else
+          V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
+                             Ld->getPointerInfo().getWithOffset(Offset), SVT,
+                             Ld->getOriginalAlign(),
+                             Ld->getMemOperand()->getFlags());
+        unsigned Opc =
+            VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
+        SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
+        return convertFromScalableVector(VT, Splat, DAG, Subtarget);
+      }
 
       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
       assert(Lane < (int)NumElts && "Unexpected lane!");
 
-      SDValue Mask, VL;
-      std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
       SDValue Gather =
           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
                       DAG.getConstant(Lane, DL, XLenVT), Mask, VL);
       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
     }
   }
 
   // Detect shuffles which can be re-expressed as vector selects.

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-vrgather.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+experimental-v,+experimental-zfh,+f,+d -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+experimental-v,+experimental-zfh,+f,+d -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
 ; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+experimental-v,+experimental-zfh,+f,+d -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+experimental-v,+experimental-zfh,+f,+d -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
 
 define void @gather_const_v8f16(<8 x half>* %x) {
 ; CHECK-LABEL: gather_const_v8f16:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    flh ft0, 10(a0)
 ; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; CHECK-NEXT:    vle16.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 5
-; CHECK-NEXT:    vse16.v v26, (a0)
+; CHECK-NEXT:    vfmv.v.f v25, ft0
+; CHECK-NEXT:    vse16.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <8 x half>, <8 x half>* %x
   %b = extractelement <8 x half> %a, i32 5
   %c = insertelement <8 x half> undef, half %b, i32 0
   %d = shufflevector <8 x half> %c, <8 x half> undef, <8 x i32> zeroinitializer
   store <8 x half> %d, <8 x half>* %x
   ret void
 }
 
 define void @gather_const_v4f32(<4 x float>* %x) {
 ; CHECK-LABEL: gather_const_v4f32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    flw ft0, 8(a0)
 ; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; CHECK-NEXT:    vle32.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 2
-; CHECK-NEXT:    vse32.v v26, (a0)
+; CHECK-NEXT:    vfmv.v.f v25, ft0
+; CHECK-NEXT:    vse32.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <4 x float>, <4 x float>* %x
   %b = extractelement <4 x float> %a, i32 2
   %c = insertelement <4 x float> undef, float %b, i32 0
   %d = shufflevector <4 x float> %c, <4 x float> undef, <4 x i32> zeroinitializer
   store <4 x float> %d, <4 x float>* %x
   ret void
 }
 
 define void @gather_const_v2f64(<2 x double>* %x) {
 ; CHECK-LABEL: gather_const_v2f64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld ft0, 0(a0)
 ; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; CHECK-NEXT:    vle64.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 0
-; CHECK-NEXT:    vse64.v v26, (a0)
+; CHECK-NEXT:    vfmv.v.f v25, ft0
+; CHECK-NEXT:    vse64.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <2 x double>, <2 x double>* %x
   %b = extractelement <2 x double> %a, i32 0
   %c = insertelement <2 x double> undef, double %b, i32 0
   %d = shufflevector <2 x double> %c, <2 x double> undef, <2 x i32> zeroinitializer
   store <2 x double> %d, <2 x double>* %x
   ret void
 }
 
 define void @gather_const_v64f16(<64 x half>* %x) {
 ; LMULMAX8-LABEL: gather_const_v64f16:
 ; LMULMAX8:       # %bb.0:
+; LMULMAX8-NEXT:    flh ft0, 94(a0)
 ; LMULMAX8-NEXT:    addi a1, zero, 64
 ; LMULMAX8-NEXT:    vsetvli a1, a1, e16,m8,ta,mu
-; LMULMAX8-NEXT:    vle16.v v8, (a0)
-; LMULMAX8-NEXT:    addi a1, zero, 47
-; LMULMAX8-NEXT:    vrgather.vx v16, v8, a1
-; LMULMAX8-NEXT:    vse16.v v16, (a0)
+; LMULMAX8-NEXT:    vfmv.v.f v8, ft0
+; LMULMAX8-NEXT:    vse16.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: gather_const_v64f16:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 80
-; LMULMAX1-NEXT:    vsetivli a2, 8, e16,m1,ta,mu
-; LMULMAX1-NEXT:    vle16.v v25, (a1)
 ; LMULMAX1-NEXT:    addi a6, a0, 16
 ; LMULMAX1-NEXT:    addi a7, a0, 48
-; LMULMAX1-NEXT:    addi a4, a0, 32
+; LMULMAX1-NEXT:    addi t0, a0, 32
+; LMULMAX1-NEXT:    addi a4, a0, 80
 ; LMULMAX1-NEXT:    addi a5, a0, 64
-; LMULMAX1-NEXT:    addi a2, a0, 112
-; LMULMAX1-NEXT:    addi a3, a0, 96
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 7
-; LMULMAX1-NEXT:    vse16.v v26, (a3)
-; LMULMAX1-NEXT:    vse16.v v26, (a2)
-; LMULMAX1-NEXT:    vse16.v v26, (a5)
-; LMULMAX1-NEXT:    vse16.v v26, (a1)
-; LMULMAX1-NEXT:    vse16.v v26, (a4)
-; LMULMAX1-NEXT:    vse16.v v26, (a7)
-; LMULMAX1-NEXT:    vse16.v v26, (a0)
-; LMULMAX1-NEXT:    vse16.v v26, (a6)
+; LMULMAX1-NEXT:    flh ft0, 94(a0)
+; LMULMAX1-NEXT:    addi a1, a0, 112
+; LMULMAX1-NEXT:    addi a2, a0, 96
+; LMULMAX1-NEXT:    vsetivli a3, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    vfmv.v.f v25, ft0
+; LMULMAX1-NEXT:    vse16.v v25, (a2)
+; LMULMAX1-NEXT:    vse16.v v25, (a1)
+; LMULMAX1-NEXT:    vse16.v v25, (a5)
+; LMULMAX1-NEXT:    vse16.v v25, (a4)
+; LMULMAX1-NEXT:    vse16.v v25, (t0)
+; LMULMAX1-NEXT:    vse16.v v25, (a7)
+; LMULMAX1-NEXT:    vse16.v v25, (a0)
+; LMULMAX1-NEXT:    vse16.v v25, (a6)
 ; LMULMAX1-NEXT:    ret
   %a = load <64 x half>, <64 x half>* %x
   %b = extractelement <64 x half> %a, i32 47
   %c = insertelement <64 x half> undef, half %b, i32 0
   %d = shufflevector <64 x half> %c, <64 x half> undef, <64 x i32> zeroinitializer
   store <64 x half> %d, <64 x half>* %x
   ret void
 }
 
 define void @gather_const_v32f32(<32 x float>* %x) {
 ; LMULMAX8-LABEL: gather_const_v32f32:
 ; LMULMAX8:       # %bb.0:
+; LMULMAX8-NEXT:    flw ft0, 68(a0)
 ; LMULMAX8-NEXT:    addi a1, zero, 32
 ; LMULMAX8-NEXT:    vsetvli a1, a1, e32,m8,ta,mu
-; LMULMAX8-NEXT:    vle32.v v8, (a0)
-; LMULMAX8-NEXT:    vrgather.vi v16, v8, 17
-; LMULMAX8-NEXT:    vse32.v v16, (a0)
+; LMULMAX8-NEXT:    vfmv.v.f v8, ft0
+; LMULMAX8-NEXT:    vse32.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: gather_const_v32f32:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 64
-; LMULMAX1-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-NEXT:    vle32.v v25, (a1)
 ; LMULMAX1-NEXT:    addi a6, a0, 16
 ; LMULMAX1-NEXT:    addi a7, a0, 48
-; LMULMAX1-NEXT:    addi a4, a0, 32
-; LMULMAX1-NEXT:    addi a5, a0, 80
-; LMULMAX1-NEXT:    addi a2, a0, 112
-; LMULMAX1-NEXT:    addi a3, a0, 96
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 1
-; LMULMAX1-NEXT:    vse32.v v26, (a3)
-; LMULMAX1-NEXT:    vse32.v v26, (a2)
-; LMULMAX1-NEXT:    vse32.v v26, (a1)
-; LMULMAX1-NEXT:    vse32.v v26, (a5)
-; LMULMAX1-NEXT:    vse32.v v26, (a4)
-; LMULMAX1-NEXT:    vse32.v v26, (a7)
-; LMULMAX1-NEXT:    vse32.v v26, (a0)
-; LMULMAX1-NEXT:    vse32.v v26, (a6)
+; LMULMAX1-NEXT:    addi t0, a0, 32
+; LMULMAX1-NEXT:    addi a4, a0, 80
+; LMULMAX1-NEXT:    addi a5, a0, 64
+; LMULMAX1-NEXT:    flw ft0, 68(a0)
+; LMULMAX1-NEXT:    addi a1, a0, 112
+; LMULMAX1-NEXT:    addi a2, a0, 96
+; LMULMAX1-NEXT:    vsetivli a3, 4, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vfmv.v.f v25, ft0
+; LMULMAX1-NEXT:    vse32.v v25, (a2)
+; LMULMAX1-NEXT:    vse32.v v25, (a1)
+; LMULMAX1-NEXT:    vse32.v v25, (a5)
+; LMULMAX1-NEXT:    vse32.v v25, (a4)
+; LMULMAX1-NEXT:    vse32.v v25, (t0)
+; LMULMAX1-NEXT:    vse32.v v25, (a7)
+; LMULMAX1-NEXT:    vse32.v v25, (a0)
+; LMULMAX1-NEXT:    vse32.v v25, (a6)
 ; LMULMAX1-NEXT:    ret
   %a = load <32 x float>, <32 x float>* %x
   %b = extractelement <32 x float> %a, i32 17
   %c = insertelement <32 x float> undef, float %b, i32 0
   %d = shufflevector <32 x float> %c, <32 x float> undef, <32 x i32> zeroinitializer
   store <32 x float> %d, <32 x float>* %x
   ret void
 }
 
 define void @gather_const_v16f64(<16 x double>* %x) {
 ; LMULMAX8-LABEL: gather_const_v16f64:
 ; LMULMAX8:       # %bb.0:
+; LMULMAX8-NEXT:    fld ft0, 80(a0)
 ; LMULMAX8-NEXT:    vsetivli a1, 16, e64,m8,ta,mu
-; LMULMAX8-NEXT:    vle64.v v8, (a0)
-; LMULMAX8-NEXT:    vrgather.vi v16, v8, 10
-; LMULMAX8-NEXT:    vse64.v v16, (a0)
+; LMULMAX8-NEXT:    vfmv.v.f v8, ft0
+; LMULMAX8-NEXT:    vse64.v v8, (a0)
 ; LMULMAX8-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: gather_const_v16f64:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 80
-; LMULMAX1-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-NEXT:    vle64.v v25, (a1)
 ; LMULMAX1-NEXT:    addi a6, a0, 16
 ; LMULMAX1-NEXT:    addi a7, a0, 48
-; LMULMAX1-NEXT:    addi a4, a0, 32
+; LMULMAX1-NEXT:    addi t0, a0, 32
+; LMULMAX1-NEXT:    addi a4, a0, 80
 ; LMULMAX1-NEXT:    addi a5, a0, 64
-; LMULMAX1-NEXT:    addi a2, a0, 112
-; LMULMAX1-NEXT:    addi a3, a0, 96
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 0
-; LMULMAX1-NEXT:    vse64.v v26, (a3)
-; LMULMAX1-NEXT:    vse64.v v26, (a2)
-; LMULMAX1-NEXT:    vse64.v v26, (a5)
-; LMULMAX1-NEXT:    vse64.v v26, (a1)
-; LMULMAX1-NEXT:    vse64.v v26, (a4)
-; LMULMAX1-NEXT:    vse64.v v26, (a7)
-; LMULMAX1-NEXT:    vse64.v v26, (a0)
-; LMULMAX1-NEXT:    vse64.v v26, (a6)
+; LMULMAX1-NEXT:    fld ft0, 80(a0)
+; LMULMAX1-NEXT:    addi a1, a0, 112
+; LMULMAX1-NEXT:    addi a2, a0, 96
+; LMULMAX1-NEXT:    vsetivli a3, 2, e64,m1,ta,mu
+; LMULMAX1-NEXT:    vfmv.v.f v25, ft0
+; LMULMAX1-NEXT:    vse64.v v25, (a2)
+; LMULMAX1-NEXT:    vse64.v v25, (a1)
+; LMULMAX1-NEXT:    vse64.v v25, (a5)
+; LMULMAX1-NEXT:    vse64.v v25, (a4)
+; LMULMAX1-NEXT:    vse64.v v25, (t0)
+; LMULMAX1-NEXT:    vse64.v v25, (a7)
+; LMULMAX1-NEXT:    vse64.v v25, (a0)
+; LMULMAX1-NEXT:    vse64.v v25, (a6)
 ; LMULMAX1-NEXT:    ret
   %a = load <16 x double>, <16 x double>* %x
   %b = extractelement <16 x double> %a, i32 10
   %c = insertelement <16 x double> undef, double %b, i32 0
   %d = shufflevector <16 x double> %c, <16 x double> undef, <16 x i32> zeroinitializer
   store <16 x double> %d, <16 x double>* %x
   ret void
 }

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-vrgather.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=4 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX4
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=4 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX4
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=4 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX4,LMULMAX4-RV32
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=4 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX4,LMULMAX4-RV64
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV32
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v -verify-machineinstrs -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1-RV64
 
 define void @gather_const_v16i8(<16 x i8>* %x) {
 ; CHECK-LABEL: gather_const_v16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 16, e8,m1,ta,mu
-; CHECK-NEXT:    vle8.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 12
-; CHECK-NEXT:    vse8.v v26, (a0)
+; CHECK-NEXT:    lb a1, 12(a0)
+; CHECK-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a1
+; CHECK-NEXT:    vse8.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <16 x i8>, <16 x i8>* %x
   %b = extractelement <16 x i8> %a, i32 12
   %c = insertelement <16 x i8> undef, i8 %b, i32 0
   %d = shufflevector <16 x i8> %c, <16 x i8> undef, <16 x i32> zeroinitializer
   store <16 x i8> %d, <16 x i8>* %x
   ret void
 }
 
 define void @gather_const_v8i16(<8 x i16>* %x) {
 ; CHECK-LABEL: gather_const_v8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
-; CHECK-NEXT:    vle16.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 5
-; CHECK-NEXT:    vse16.v v26, (a0)
+; CHECK-NEXT:    lh a1, 10(a0)
+; CHECK-NEXT:    vsetivli a2, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a1
+; CHECK-NEXT:    vse16.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <8 x i16>, <8 x i16>* %x
   %b = extractelement <8 x i16> %a, i32 5
   %c = insertelement <8 x i16> undef, i16 %b, i32 0
   %d = shufflevector <8 x i16> %c, <8 x i16> undef, <8 x i32> zeroinitializer
   store <8 x i16> %d, <8 x i16>* %x
   ret void
 }
 
 define void @gather_const_v4i32(<4 x i32>* %x) {
 ; CHECK-LABEL: gather_const_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 4, e32,m1,ta,mu
-; CHECK-NEXT:    vle32.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 3
-; CHECK-NEXT:    vse32.v v26, (a0)
+; CHECK-NEXT:    lw a1, 12(a0)
+; CHECK-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a1
+; CHECK-NEXT:    vse32.v v25, (a0)
 ; CHECK-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x
   %b = extractelement <4 x i32> %a, i32 3
   %c = insertelement <4 x i32> undef, i32 %b, i32 0
   %d = shufflevector <4 x i32> %c, <4 x i32> undef, <4 x i32> zeroinitializer
   store <4 x i32> %d, <4 x i32>* %x
   ret void
 }
 
 define void @gather_const_v2i64(<2 x i64>* %x) {
-; CHECK-LABEL: gather_const_v2i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
-; CHECK-NEXT:    vle64.v v25, (a0)
-; CHECK-NEXT:    vrgather.vi v26, v25, 1
-; CHECK-NEXT:    vse64.v v26, (a0)
-; CHECK-NEXT:    ret
+; LMULMAX4-RV32-LABEL: gather_const_v2i64:
+; LMULMAX4-RV32:       # %bb.0:
+; LMULMAX4-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX4-RV32-NEXT:    vle64.v v25, (a0)
+; LMULMAX4-RV32-NEXT:    vrgather.vi v26, v25, 1
+; LMULMAX4-RV32-NEXT:    vse64.v v26, (a0)
+; LMULMAX4-RV32-NEXT:    ret
+;
+; LMULMAX4-RV64-LABEL: gather_const_v2i64:
+; LMULMAX4-RV64:       # %bb.0:
+; LMULMAX4-RV64-NEXT:    ld a1, 8(a0)
+; LMULMAX4-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX4-RV64-NEXT:    vmv.v.x v25, a1
+; LMULMAX4-RV64-NEXT:    vse64.v v25, (a0)
+; LMULMAX4-RV64-NEXT:    ret
+;
+; LMULMAX1-RV32-LABEL: gather_const_v2i64:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli a1, 2, e64,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vle64.v v25, (a0)
+; LMULMAX1-RV32-NEXT:    vrgather.vi v26, v25, 1
+; LMULMAX1-RV32-NEXT:    vse64.v v26, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: gather_const_v2i64:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    ld a1, 8(a0)
+; LMULMAX1-RV64-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v25, a1
+; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x
   %b = extractelement <2 x i64> %a, i32 1
   %c = insertelement <2 x i64> undef, i64 %b, i32 0
   %d = shufflevector <2 x i64> %c, <2 x i64> undef, <2 x i32> zeroinitializer
   store <2 x i64> %d, <2 x i64>* %x
   ret void
 }
 
 define void @gather_const_v64i8(<64 x i8>* %x) {
 ; LMULMAX4-LABEL: gather_const_v64i8:
 ; LMULMAX4:       # %bb.0:
-; LMULMAX4-NEXT:    addi a1, zero, 64
-; LMULMAX4-NEXT:    vsetvli a1, a1, e8,m4,ta,mu
-; LMULMAX4-NEXT:    vle8.v v28, (a0)
-; LMULMAX4-NEXT:    addi a1, zero, 32
-; LMULMAX4-NEXT:    vrgather.vx v8, v28, a1
-; LMULMAX4-NEXT:    vse8.v v8, (a0)
+; LMULMAX4-NEXT:    lb a1, 32(a0)
+; LMULMAX4-NEXT:    addi a2, zero, 64
+; LMULMAX4-NEXT:    vsetvli a2, a2, e8,m4,ta,mu
+; LMULMAX4-NEXT:    vmv.v.x v28, a1
+; LMULMAX4-NEXT:    vse8.v v28, (a0)
 ; LMULMAX4-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: gather_const_v64i8:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 32
-; LMULMAX1-NEXT:    vsetivli a2, 16, e8,m1,ta,mu
-; LMULMAX1-NEXT:    vle8.v v25, (a1)
-; LMULMAX1-NEXT:    addi a2, a0, 16
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    lb a2, 32(a0)
 ; LMULMAX1-NEXT:    addi a3, a0, 48
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 0
-; LMULMAX1-NEXT:    vse8.v v26, (a1)
-; LMULMAX1-NEXT:    vse8.v v26, (a3)
-; LMULMAX1-NEXT:    vse8.v v26, (a0)
-; LMULMAX1-NEXT:    vse8.v v26, (a2)
+; LMULMAX1-NEXT:    addi a4, a0, 32
+; LMULMAX1-NEXT:    vsetivli a5, 16, e8,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.x v25, a2
+; LMULMAX1-NEXT:    vse8.v v25, (a4)
+; LMULMAX1-NEXT:    vse8.v v25, (a3)
+; LMULMAX1-NEXT:    vse8.v v25, (a0)
+; LMULMAX1-NEXT:    vse8.v v25, (a1)
 ; LMULMAX1-NEXT:    ret
   %a = load <64 x i8>, <64 x i8>* %x
   %b = extractelement <64 x i8> %a, i32 32
   %c = insertelement <64 x i8> undef, i8 %b, i32 0
   %d = shufflevector <64 x i8> %c, <64 x i8> undef, <64 x i32> zeroinitializer
   store <64 x i8> %d, <64 x i8>* %x
   ret void
 }
 
 define void @gather_const_v16i16(<32 x i16>* %x) {
 ; LMULMAX4-LABEL: gather_const_v16i16:
 ; LMULMAX4:       # %bb.0:
-; LMULMAX4-NEXT:    addi a1, zero, 32
-; LMULMAX4-NEXT:    vsetvli a1, a1, e16,m4,ta,mu
-; LMULMAX4-NEXT:    vle16.v v28, (a0)
-; LMULMAX4-NEXT:    vrgather.vi v8, v28, 25
-; LMULMAX4-NEXT:    vse16.v v8, (a0)
+; LMULMAX4-NEXT:    lh a1, 50(a0)
+; LMULMAX4-NEXT:    addi a2, zero, 32
+; LMULMAX4-NEXT:    vsetvli a2, a2, e16,m4,ta,mu
+; LMULMAX4-NEXT:    vmv.v.x v28, a1
+; LMULMAX4-NEXT:    vse16.v v28, (a0)
 ; LMULMAX4-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: gather_const_v16i16:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 48
-; LMULMAX1-NEXT:    vsetivli a2, 8, e16,m1,ta,mu
-; LMULMAX1-NEXT:    vle16.v v25, (a1)
-; LMULMAX1-NEXT:    addi a2, a0, 16
-; LMULMAX1-NEXT:    addi a3, a0, 32
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 1
-; LMULMAX1-NEXT:    vse16.v v26, (a3)
-; LMULMAX1-NEXT:    vse16.v v26, (a1)
-; LMULMAX1-NEXT:    vse16.v v26, (a0)
-; LMULMAX1-NEXT:    vse16.v v26, (a2)
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    lh a2, 50(a0)
+; LMULMAX1-NEXT:    addi a3, a0, 48
+; LMULMAX1-NEXT:    addi a4, a0, 32
+; LMULMAX1-NEXT:    vsetivli a5, 8, e16,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.x v25, a2
+; LMULMAX1-NEXT:    vse16.v v25, (a4)
+; LMULMAX1-NEXT:    vse16.v v25, (a3)
+; LMULMAX1-NEXT:    vse16.v v25, (a0)
+; LMULMAX1-NEXT:    vse16.v v25, (a1)
 ; LMULMAX1-NEXT:    ret
   %a = load <32 x i16>, <32 x i16>* %x
   %b = extractelement <32 x i16> %a, i32 25
   %c = insertelement <32 x i16> undef, i16 %b, i32 0
   %d = shufflevector <32 x i16> %c, <32 x i16> undef, <32 x i32> zeroinitializer
   store <32 x i16> %d, <32 x i16>* %x
   ret void
 }
 
 define void @gather_const_v16i32(<16 x i32>* %x) {
 ; LMULMAX4-LABEL: gather_const_v16i32:
 ; LMULMAX4:       # %bb.0:
-; LMULMAX4-NEXT:    vsetivli a1, 16, e32,m4,ta,mu
-; LMULMAX4-NEXT:    vle32.v v28, (a0)
-; LMULMAX4-NEXT:    vrgather.vi v8, v28, 9
-; LMULMAX4-NEXT:    vse32.v v8, (a0)
+; LMULMAX4-NEXT:    lw a1, 36(a0)
+; LMULMAX4-NEXT:    vsetivli a2, 16, e32,m4,ta,mu
+; LMULMAX4-NEXT:    vmv.v.x v28, a1
+; LMULMAX4-NEXT:    vse32.v v28, (a0)
 ; LMULMAX4-NEXT:    ret
 ;
 ; LMULMAX1-LABEL: gather_const_v16i32:
 ; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 32
-; LMULMAX1-NEXT:    vsetivli a2, 4, e32,m1,ta,mu
-; LMULMAX1-NEXT:    vle32.v v25, (a1)
-; LMULMAX1-NEXT:    addi a2, a0, 16
+; LMULMAX1-NEXT:    addi a1, a0, 16
+; LMULMAX1-NEXT:    lw a2, 36(a0)
 ; LMULMAX1-NEXT:    addi a3, a0, 48
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 1
-; LMULMAX1-NEXT:    vse32.v v26, (a1)
-; LMULMAX1-NEXT:    vse32.v v26, (a3)
-; LMULMAX1-NEXT:    vse32.v v26, (a0)
-; LMULMAX1-NEXT:    vse32.v v26, (a2)
+; LMULMAX1-NEXT:    addi a4, a0, 32
+; LMULMAX1-NEXT:    vsetivli a5, 4, e32,m1,ta,mu
+; LMULMAX1-NEXT:    vmv.v.x v25, a2
+; LMULMAX1-NEXT:    vse32.v v25, (a4)
+; LMULMAX1-NEXT:    vse32.v v25, (a3)
+; LMULMAX1-NEXT:    vse32.v v25, (a0)
+; LMULMAX1-NEXT:    vse32.v v25, (a1)
 ; LMULMAX1-NEXT:    ret
   %a = load <16 x i32>, <16 x i32>* %x
   %b = extractelement <16 x i32> %a, i32 9
   %c = insertelement <16 x i32> undef, i32 %b, i32 0
   %d = shufflevector <16 x i32> %c, <16 x i32> undef, <16 x i32> zeroinitializer
   store <16 x i32> %d, <16 x i32>* %x
   ret void
 }
 
 define void @gather_const_v8i64(<8 x i64>* %x) {
-; LMULMAX4-LABEL: gather_const_v8i64:
-; LMULMAX4:       # %bb.0:
-; LMULMAX4-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
-; LMULMAX4-NEXT:    vle64.v v28, (a0)
-; LMULMAX4-NEXT:    vrgather.vi v8, v28, 3
-; LMULMAX4-NEXT:    vse64.v v8, (a0)
-; LMULMAX4-NEXT:    ret
+; LMULMAX4-RV32-LABEL: gather_const_v8i64:
+; LMULMAX4-RV32:       # %bb.0:
+; LMULMAX4-RV32-NEXT:    vsetivli a1, 8, e64,m4,ta,mu
+; LMULMAX4-RV32-NEXT:    vle64.v v28, (a0)
+; LMULMAX4-RV32-NEXT:    vrgather.vi v8, v28, 3
+; LMULMAX4-RV32-NEXT:    vse64.v v8, (a0)
+; LMULMAX4-RV32-NEXT:    ret
 ;
-; LMULMAX1-LABEL: gather_const_v8i64:
-; LMULMAX1:       # %bb.0:
-; LMULMAX1-NEXT:    addi a1, a0, 16
-; LMULMAX1-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
-; LMULMAX1-NEXT:    vle64.v v25, (a1)
-; LMULMAX1-NEXT:    addi a2, a0, 48
-; LMULMAX1-NEXT:    addi a3, a0, 32
-; LMULMAX1-NEXT:    vrgather.vi v26, v25, 1
-; LMULMAX1-NEXT:    vse64.v v26, (a3)
-; LMULMAX1-NEXT:    vse64.v v26, (a2)
-; LMULMAX1-NEXT:    vse64.v v26, (a0)
-; LMULMAX1-NEXT:    vse64.v v26, (a1)
-; LMULMAX1-NEXT:    ret
+; LMULMAX4-RV64-LABEL: gather_const_v8i64:
+; LMULMAX4-RV64:       # %bb.0:
+; LMULMAX4-RV64-NEXT:    ld a1, 24(a0)
+; LMULMAX4-RV64-NEXT:    vsetivli a2, 8, e64,m4,ta,mu
+; LMULMAX4-RV64-NEXT:    vmv.v.x v28, a1
+; LMULMAX4-RV64-NEXT:    vse64.v v28, (a0)
+; LMULMAX4-RV64-NEXT:    ret
+;
+; LMULMAX1-RV32-LABEL: gather_const_v8i64:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
+; LMULMAX1-RV32-NEXT:    vsetivli a2, 2, e64,m1,ta,mu
+; LMULMAX1-RV32-NEXT:    vle64.v v25, (a1)
+; LMULMAX1-RV32-NEXT:    addi a2, a0, 48
+; LMULMAX1-RV32-NEXT:    addi a3, a0, 32
+; LMULMAX1-RV32-NEXT:    vrgather.vi v26, v25, 1
+; LMULMAX1-RV32-NEXT:    vse64.v v26, (a3)
+; LMULMAX1-RV32-NEXT:    vse64.v v26, (a2)
+; LMULMAX1-RV32-NEXT:    vse64.v v26, (a0)
+; LMULMAX1-RV32-NEXT:    vse64.v v26, (a1)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: gather_const_v8i64:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
+; LMULMAX1-RV64-NEXT:    ld a2, 24(a0)
+; LMULMAX1-RV64-NEXT:    addi a3, a0, 48
+; LMULMAX1-RV64-NEXT:    addi a4, a0, 32
+; LMULMAX1-RV64-NEXT:    vsetivli a5, 2, e64,m1,ta,mu
+; LMULMAX1-RV64-NEXT:    vmv.v.x v25, a2
+; LMULMAX1-RV64-NEXT:    vse64.v v25, (a4)
+; LMULMAX1-RV64-NEXT:    vse64.v v25, (a3)
+; LMULMAX1-RV64-NEXT:    vse64.v v25, (a0)
+; LMULMAX1-RV64-NEXT:    vse64.v v25, (a1)
+; LMULMAX1-RV64-NEXT:    ret
   %a = load <8 x i64>, <8 x i64>* %x
   %b = extractelement <8 x i64> %a, i32 3
   %c = insertelement <8 x i64> undef, i64 %b, i32 0
   %d = shufflevector <8 x i64> %c, <8 x i64> undef, <8 x i32> zeroinitializer
   store <8 x i64> %d, <8 x i64>* %x
   ret void
 }
+
+define void @splat_concat_low(<4 x i16>* %x, <4 x i16>* %y, <8 x i16>* %z) {
+; CHECK-LABEL: splat_concat_low:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lh a0, 0(a0)
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a0
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %a = load <4 x i16>, <4 x i16>* %x
+  %b = load <4 x i16>, <4 x i16>* %y
+  %c = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %d = shufflevector <8 x i16> %c, <8 x i16> undef, <8 x i32> zeroinitializer
+  store <8 x i16> %d, <8 x i16>* %z
+  ret void
+}
+
+define void @splat_concat_high(<4 x i16>* %x, <4 x i16>* %y, <8 x i16>* %z) {
+; CHECK-LABEL: splat_concat_high:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lh a0, 2(a1)
+; CHECK-NEXT:    vsetivli a1, 8, e16,m1,ta,mu
+; CHECK-NEXT:    vmv.v.x v25, a0
+; CHECK-NEXT:    vse16.v v25, (a2)
+; CHECK-NEXT:    ret
+  %a = load <4 x i16>, <4 x i16>* %x
+  %b = load <4 x i16>, <4 x i16>* %y
+  %c = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %d = shufflevector <8 x i16> %c, <8 x i16> undef, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  store <8 x i16> %d, <8 x i16>* %z
+  ret void
+}