[RISCV] Fold store of vmv.x.s to a vse with VL=1.

[craig.topper]

Summary

This can avoid a loss of decoupling with the scalar unit on cores
with decoupled scalar and vector units.

We should support FP too, but those use extract_element and not a
custom ISD node so it is a little different. I also left a FIXME
in the test for i64 extract and store on RV32.

[frasercrmck]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

7132

I was wondering what'd happen if you replaced it with a ISD::VP_STORE. Just thinking that a standard node would potentially provide more optimization opportunities compared to a RISCVISD node.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

7132

If I'm reading the code right, we don't recognize VP_STORE with an all ones mask as special and I think we'll end up lowering it to the vse_mask intrinsic.

[craig.topper]

Use VP_STORE. I couldn't copy the MachineMemOperand and had to have getStoreVP rebuild it. This is due to getStoreVP not taking the MemVT as an argument. This caused an assert if the MemOperand size, which was correctly one element, didn't match the VP_STORE VT. So now we have a MemOperand with an unknown size I guess.

[frasercrmck]

LGTM but I think you have a couple of test failures there?

[craig.topper]

In D109482#3023981, @frasercrmck wrote:

LGTM but I think you have a couple of test failures there?

Those may have ran while the vp.store with all ones mask patch was still outstanding. Everything passes with ninja check-llvm right now.

llvm/include/llvm/CodeGen/SelectionDAG.h

SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
                        SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL,
                        EVT MemVT, MachineMemOperand *MMO,
                        bool IsExpanding = false);
   SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
                            SDValue Offset, ISD::MemIndexedMode AM);
   SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
                      SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo,
                      Align Alignment, MachineMemOperand::Flags MMOFlags,
-                     const AAMDNodes &AAInfo, bool IsCompressing = false);
+                     const AAMDNodes &AAInfo = AAMDNodes(),
+                     bool IsCompressing = false);
   SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
                      SDValue Mask, SDValue EVL, MachineMemOperand *MMO,
                      bool IsCompressing = false);
   SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
                           SDValue Ptr, SDValue Mask, SDValue EVL,
                           MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
                           MachineMemOperand::Flags MMOFlags,
                           const AAMDNodes &AAInfo, bool IsCompressing = false);

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (Subtarget.hasStdExtV()) {
     setTargetDAGCombine(ISD::FCOPYSIGN);
     setTargetDAGCombine(ISD::MGATHER);
     setTargetDAGCombine(ISD::MSCATTER);
     setTargetDAGCombine(ISD::VP_GATHER);
     setTargetDAGCombine(ISD::VP_SCATTER);
     setTargetDAGCombine(ISD::SRA);
     setTargetDAGCombine(ISD::SRL);
     setTargetDAGCombine(ISD::SHL);
+    setTargetDAGCombine(ISD::STORE);
   }
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
                                             LLVMContext &Context,
                                             EVT VT) const {
   if (!VT.isVector())
     return getPointerTy(DL);
case RISCVISD::MUL_VL: {
     SDValue Op0 = N->getOperand(0);
     SDValue Op1 = N->getOperand(1);
     if (SDValue V = combineMUL_VLToVWMUL(N, Op0, Op1, DAG))
       return V;
     if (SDValue V = combineMUL_VLToVWMUL(N, Op1, Op0, DAG))
       return V;
     return SDValue();
   }
+  case ISD::STORE: {
+    auto *Store = cast<StoreSDNode>(N);
+    SDValue Val = Store->getValue();
+    // Combine store of vmv.x.s to vse with VL of 1.
+    // FIXME: Support FP.
+    if (Val.getOpcode() == RISCVISD::VMV_X_S) {
+      SDValue Src = Val.getOperand(0);
+      EVT VecVT = Src.getValueType();
+      EVT MemVT = Store->getMemoryVT();
+      // The memory VT and the element type must match.
+      if (VecVT.getVectorElementType() == MemVT) {
+        SDLoc DL(N);
+        MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
+        return DAG.getStoreVP(Store->getChain(), DL, Src, Store->getBasePtr(),
+                              DAG.getConstant(1, DL, MaskVT),
+                              DAG.getConstant(1, DL, Subtarget.getXLenVT()),
+                              Store->getPointerInfo(),
+                              Store->getOriginalAlign(),
+                              Store->getMemOperand()->getFlags());
+      }
+    }
+
+    break;
+  }
   }
 
   return SDValue();
 }
 
 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
     const SDNode *N, CombineLevel Level) const {
   // The following folds are only desirable if `(OP _, c1 << c2)` can be

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll

 ; RV64-NEXT:    vslidedown.vx v26, v26, a1
 ; RV64-NEXT:    vmv.x.s a0, v26
 ; RV64-NEXT:    ret
   %a = load <3 x i64>, <3 x i64>* %x
   %b = add <3 x i64> %a, %a
   %c = extractelement <3 x i64> %b, i32 %idx
   ret i64 %c
 }
+
+define void @store_extractelt_v16i8(<16 x i8>* %x, i8* %p) nounwind {
+; CHECK-LABEL: store_extractelt_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; CHECK-NEXT:    vle8.v v25, (a0)
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v25, v25, 7
+; CHECK-NEXT:    vse8.v v25, (a1)
+; CHECK-NEXT:    ret
+  %a = load <16 x i8>, <16 x i8>* %x
+  %b = extractelement <16 x i8> %a, i32 7
+  store i8 %b, i8* %p
+  ret void
+}
+
+define void @store_extractelt_v8i16(<8 x i16>* %x, i16* %p) nounwind {
+; CHECK-LABEL: store_extractelt_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT:    vle16.v v25, (a0)
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v25, v25, 7
+; CHECK-NEXT:    vse16.v v25, (a1)
+; CHECK-NEXT:    ret
+  %a = load <8 x i16>, <8 x i16>* %x
+  %b = extractelement <8 x i16> %a, i32 7
+  store i16 %b, i16* %p
+  ret void
+}
+
+define void @store_extractelt_v4i32(<4 x i32>* %x, i32* %p) nounwind {
+; CHECK-LABEL: store_extractelt_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
+; CHECK-NEXT:    vle32.v v25, (a0)
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, mu
+; CHECK-NEXT:    vslidedown.vi v25, v25, 2
+; CHECK-NEXT:    vse32.v v25, (a1)
+; CHECK-NEXT:    ret
+  %a = load <4 x i32>, <4 x i32>* %x
+  %b = extractelement <4 x i32> %a, i32 2
+  store i32 %b, i32* %p
+  ret void
+}
+
+; FIXME: Use vse64.v on RV32 to avoid two scalar extracts and two scalar stores.
+define void @store_extractelt_v4i64(<2 x i64>* %x, i64* %p) nounwind {
+; RV32-LABEL: store_extractelt_v4i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; RV32-NEXT:    vle64.v v25, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
+; RV32-NEXT:    vslidedown.vi v25, v25, 1
+; RV32-NEXT:    addi a0, zero, 32
+; RV32-NEXT:    vsrl.vx v26, v25, a0
+; RV32-NEXT:    vmv.x.s a0, v26
+; RV32-NEXT:    vmv.x.s a2, v25
+; RV32-NEXT:    sw a2, 0(a1)
+; RV32-NEXT:    sw a0, 4(a1)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: store_extractelt_v4i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
+; RV64-NEXT:    vle64.v v25, (a0)
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
+; RV64-NEXT:    vslidedown.vi v25, v25, 1
+; RV64-NEXT:    vse64.v v25, (a1)
+; RV64-NEXT:    ret
+  %a = load <2 x i64>, <2 x i64>* %x
+  %b = extractelement <2 x i64> %a, i64 1
+  store i64 %b, i64* %p
+  ret void
+}
+

llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll

 ; RV32MV-NEXT:    addi a0, a0, %lo(.LCPI3_0)
 ; RV32MV-NEXT:    vle32.v v30, (a0)
 ; RV32MV-NEXT:    vmerge.vim v26, v26, -1, v0
 ; RV32MV-NEXT:    vand.vv v26, v28, v26
 ; RV32MV-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
 ; RV32MV-NEXT:    vmsne.vv v0, v26, v30
 ; RV32MV-NEXT:    vmv.v.i v26, 0
 ; RV32MV-NEXT:    vmerge.vim v26, v26, -1, v0
-; RV32MV-NEXT:    vsetivli zero, 0, e32, m2, ta, mu
-; RV32MV-NEXT:    vmv.x.s a0, v26
-; RV32MV-NEXT:    sw a0, 0(s1)
 ; RV32MV-NEXT:    vsetivli zero, 1, e32, m2, ta, mu
+; RV32MV-NEXT:    vse32.v v26, (s1)
 ; RV32MV-NEXT:    vslidedown.vi v28, v26, 1
 ; RV32MV-NEXT:    vmv.x.s a0, v28
 ; RV32MV-NEXT:    vslidedown.vi v28, v26, 2
 ; RV32MV-NEXT:    vmv.x.s a1, v28
 ; RV32MV-NEXT:    slli a2, a1, 1
 ; RV32MV-NEXT:    sub a0, a2, a0
 ; RV32MV-NEXT:    sw a0, 4(s1)
 ; RV32MV-NEXT:    vslidedown.vi v28, v26, 4