[AArch64][SVE] Optimize mov and sel away for masked loads in sel

[MattDevereau]

Summary

fold (sel (p) (masked_load (ptr) (p)) passthru) -> (masked_load (p) (ptr))

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static SDValue trySwapVSelectOperands(SDNode *N, SelectionDAG &DAG) {
   auto InverseSetCC = DAG.getSetCC(
       SDLoc(SetCC), SetCC.getValueType(), SetCC.getOperand(0),
       SetCC.getOperand(1), ISD::getSetCCInverse(CC, SetCC.getValueType()));
 
   return DAG.getNode(ISD::VSELECT, SDLoc(N), NTy,
                      {InverseSetCC, SelectB, SelectA});
 }
 
+static SDValue tryFoldVSelectMaskedLoad(SDNode *N, SelectionDAG &DAG) {
+  auto VSelectLHS = N->getOperand(1);
+  if (VSelectLHS.getOpcode() != ISD::MLOAD)
+    return SDValue();
+
+  APInt SplatValue;
+  if (!ISD::isConstantSplatVector(VSelectLHS.getOperand(4).getNode(),
+                                  SplatValue))
+    return SDValue();
+  if (!SplatValue.isZero())
+    return SDValue();
+
+  return VSelectLHS;
+}
+
 // vselect (v1i1 setcc) ->
 //     vselect (v1iXX setcc)  (XX is the size of the compared operand type)
 // FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as
 // condition. If it can legalize "VSELECT v1i1" correctly, no need to combine
 // such VSELECT.
 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
   if (auto SwapResult = trySwapVSelectOperands(N, DAG))
     return SwapResult;
+  if (auto FoldMaskedLoadResult = tryFoldVSelectMaskedLoad(N, DAG))
+    return FoldMaskedLoadResult;
 
   SDValue N0 = N->getOperand(0);
   EVT CCVT = N0.getValueType();
 
   if (isAllActivePredicate(DAG, N0))
     return N->getOperand(1);
 
   if (isAllInactivePredicate(N0))

llvm/test/CodeGen/AArch64/sve-select.ll

 ; CHECK-NEXT:    fmul v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    ret
   %p = fcmp oeq <4 x float> %a, zeroinitializer
   %fmul = fmul <4 x float> %a, %b
   %sel = select <4 x i1> %p, <4 x float> %a, <4 x float> %fmul
   ret <4 x float> %sel
 }
+
+define <vscale x 4 x i32> @fold_vselect_masked_load_zero(i32* %ptr) {
+; CHECK-LABEL: fold_vselect_masked_load_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl16
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
+; CHECK-NEXT:    ret
+  %p = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 9)
+  %vscaleptr = bitcast i32* %ptr to <vscale x 4 x i32>*
+  %load = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>* %vscaleptr, i32 1, <vscale x 4 x i1> %p, <vscale x 4 x i32> zeroinitializer)
+  %sel = select <vscale x 4 x i1> %p, <vscale x 4 x i32> %load, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x i32> %sel
+}
+
+declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 immarg)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sel.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>*, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
 No newline at end of file