[RISCV] Fold store of vmv.x.s to a vse with VL=1.

[craig.topper]

Summary

This can avoid a loss of decoupling with the scalar unit on cores
with decoupled scalar and vector units.

We should support FP too, but those use extract_element and not a
custom ISD node so it is a little different. I also left a FIXME
in the test for i64 extract and store on RV32.

[frasercrmck]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

6596

I was wondering what'd happen if you replaced it with a ISD::VP_STORE. Just thinking that a standard node would potentially provide more optimization opportunities compared to a RISCVISD node.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

6596

If I'm reading the code right, we don't recognize VP_STORE with an all ones mask as special and I think we'll end up lowering it to the vse_mask intrinsic.

[craig.topper]

Use VP_STORE. I couldn't copy the MachineMemOperand and had to have getStoreVP rebuild it. This is due to getStoreVP not taking the MemVT as an argument. This caused an assert if the MemOperand size, which was correctly one element, didn't match the VP_STORE VT. So now we have a MemOperand with an unknown size I guess.

[frasercrmck]

LGTM but I think you have a couple of test failures there?

[craig.topper]

In D109482#3023981, @frasercrmck wrote:

LGTM but I think you have a couple of test failures there?

Those may have ran while the vp.store with all ones mask patch was still outstanding. Everything passes with ninja check-llvm right now.

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
if (Subtarget.hasStdExtV()) {
     setTargetDAGCombine(ISD::FCOPYSIGN);
     setTargetDAGCombine(ISD::MGATHER);
     setTargetDAGCombine(ISD::MSCATTER);
     setTargetDAGCombine(ISD::VP_GATHER);
     setTargetDAGCombine(ISD::VP_SCATTER);
     setTargetDAGCombine(ISD::SRA);
     setTargetDAGCombine(ISD::SRL);
     setTargetDAGCombine(ISD::SHL);
+    setTargetDAGCombine(ISD::STORE);
   }
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
                                             LLVMContext &Context,
                                             EVT VT) const {
   if (!VT.isVector())
     return getPointerTy(DL);
case RISCVISD::MUL_VL: {
     if (Op0.getValueType() != NarrowVT)
       Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
     if (Op1.getValueType() != NarrowVT)
       Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
 
     unsigned WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
     return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
   }
+  case ISD::STORE: {
+    auto *Store = cast<StoreSDNode>(N);
+    SDValue Val = Store->getValue();
+    // Combine store of vmv.x.s to vse with VL of 1.
+    // FIXME: Support FP.
+    if (Val.getOpcode() == RISCVISD::VMV_X_S) {
+      SDValue Src = Val.getOperand(0);
+      EVT VecVT = Src.getValueType();
+      EVT MemVT = Store->getMemoryVT();
+      // The memory VT and the element type must match.
+      if (VecVT.getVectorElementType() == MemVT) {
+        SDLoc DL(N);
+        return DAG.getMemIntrinsicNode(
+            RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
+            {Store->getChain(), Src, Store->getBasePtr(),
+             DAG.getConstant(1, DL, Subtarget.getXLenVT())},
+            Store->getMemoryVT(), Store->getMemOperand());
+      }
+    }
+
+    break;
+  }
   }
 
   return SDValue();
 }
 
 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
     const SDNode *N, CombineLevel Level) const {
   // The following folds are only desirable if `(OP _, c1 << c2)` can be

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll

 ; RV64-NEXT:    vslidedown.vx v26, v26, a1
 ; RV64-NEXT:    vmv.x.s a0, v26
 ; RV64-NEXT:    ret
   %a = load <3 x i64>, <3 x i64>* %x
   %b = add <3 x i64> %a, %a
   %c = extractelement <3 x i64> %b, i32 %idx
   ret i64 %c
 }
+
+define void @store_extractelt_v16i8(<16 x i8>* %x, i8* %p) nounwind {
+; CHECK-LABEL: store_extractelt_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; CHECK-NEXT:    vle8.v v25, (a0)
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v25, v25, 7
+; CHECK-NEXT:    vse8.v v25, (a1)
+; CHECK-NEXT:    ret
+  %a = load <16 x i8>, <16 x i8>* %x
+  %b = extractelement <16 x i8> %a, i32 7
+  store i8 %b, i8* %p
+  ret void
+}
+
+define void @store_extractelt_v8i16(<8 x i16>* %x, i16* %p) nounwind {
+; CHECK-LABEL: store_extractelt_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT:    vle16.v v25, (a0)
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v25, v25, 7
+; CHECK-NEXT:    vse16.v v25, (a1)
+; CHECK-NEXT:    ret
+  %a = load <8 x i16>, <8 x i16>* %x
+  %b = extractelement <8 x i16> %a, i32 7
+  store i16 %b, i16* %p
+  ret void
+}
+
+define void @store_extractelt_v4i32(<4 x i32>* %x, i32* %p) nounwind {
+; CHECK-LABEL: store_extractelt_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v25, v25, 2
+; CHECK-NEXT:    vse32.v v25, (a1)
+; CHECK-NEXT:    ret
+  %a = load <4 x i32>, <4 x i32>* %x
+  %b = extractelement <4 x i32> %a, i32 2
+  store i32 %b, i32* %p
+  ret void
+}
+
+; FIXME: Use vse64.v on RV32 to avoid two scalar extracts and two scalar stores.
+define void @store_extractelt_v4i64(<2 x i64>* %x, i64* %p) nounwind {
+; RV32-LABEL: store_extractelt_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
+; RV32-NEXT:    vslidedown.vi v25, v25, 1
+; RV32-NEXT:    addi a0, zero, 32
+; RV32-NEXT:    vsrl.vx v26, v25, a0
+; RV32-NEXT:    vmv.x.s a0, v26
+; RV32-NEXT:    vmv.x.s a2, v25
+; RV32-NEXT:    sw a2, 0(a1)
+; RV32-NEXT:    sw a0, 4(a1)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: store_extractelt_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
+; RV64-NEXT:    vslidedown.vi v25, v25, 1
+; RV64-NEXT:    vse64.v v25, (a1)
+; RV64-NEXT:    ret
+  %a = load <2 x i64>, <2 x i64>* %x
+  %b = extractelement <2 x i64> %a, i64 1
+  store i64 %b, i64* %p
+  ret void
+}
+

llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll

 ; RV32MV-NEXT:    addi a0, a0, %lo(.LCPI3_0)
 ; RV32MV-NEXT:    vle32.v v30, (a0)
 ; RV32MV-NEXT:    vmerge.vim v26, v26, -1, v0
 ; RV32MV-NEXT:    vand.vv v26, v28, v26
 ; RV32MV-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
 ; RV32MV-NEXT:    vmsne.vv v0, v26, v30
 ; RV32MV-NEXT:    vmv.v.i v26, 0
 ; RV32MV-NEXT:    vmerge.vim v26, v26, -1, v0
-; RV32MV-NEXT:    vsetivli zero, 0, e32, m2, ta, mu
-; RV32MV-NEXT:    vmv.x.s a0, v26
-; RV32MV-NEXT:    sw a0, 0(s1)
 ; RV32MV-NEXT:    vsetivli zero, 1, e32, m2, ta, mu
+; RV32MV-NEXT:    vse32.v v26, (s1)
 ; RV32MV-NEXT:    vslidedown.vi v28, v26, 1
 ; RV32MV-NEXT:    vmv.x.s a0, v28
 ; RV32MV-NEXT:    vslidedown.vi v28, v26, 2
 ; RV32MV-NEXT:    vmv.x.s a1, v28
 ; RV32MV-NEXT:    slli a2, a1, 1
 ; RV32MV-NEXT:    sub a0, a2, a0
 ; RV32MV-NEXT:    sw a0, 4(s1)
 ; RV32MV-NEXT:    vslidedown.vi v28, v26, 4