[DAGCombiner] Replace gathers with a zero mask with the passthru value

[bkramer]

Summary

These can be created by the legalizer when splitting a larger gather.
Also make isNullOrNullSplat a bit smarter by ignoring truncation in
BUILD_VECTORs.

[craig.topper]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

8092

I wonder if we should just use ISD::isBuildVectorAllZeros here.

llvm/test/CodeGen/AArch64/vecreduce-bool.ll

100

I assume this was caused by the isNullOrNullSplat change?

[bkramer]

Use isBuildVectorAllZeros, drop isNullOrNullSplat change.

[bkramer]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

8092

That's indeed simpler. Why do we have so many of these things? :(

llvm/test/CodeGen/AArch64/vecreduce-bool.ll

100

Yeah. It's a minor improvement. But I dropped this for now.

[craig.topper]

LGTM

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

if (Mask.getOpcode() == ISD::SETCC) {
     AddToWorklist(Hi.getNode());
 
     return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
   }
   return SDValue();
 }
 
 SDValue DAGCombiner::visitMGATHER(SDNode *N) {
-  if (Level >= AfterLegalizeTypes)
-    return SDValue();
-
   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
   SDValue Mask = MGT->getMask();
   SDLoc DL(N);
 
+  // Zap gathers with a zero mask.
+  if (isNullOrNullSplat(Mask))
+    return CombineTo(N, MGT->getPassThru(), MGT->getChain());
+
+  if (Level >= AfterLegalizeTypes)
+    return SDValue();
+
   // If the MGATHER result requires splitting and the mask is provided by a
   // SETCC, then split both nodes and its operands before legalization. This
   // prevents the type legalizer from unrolling SETCC into scalar comparisons
   // and enables future optimizations (e.g. min/max pattern matching on X86).
 
   if (Mask.getOpcode() != ISD::SETCC)
     return SDValue();
 

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

 
 bool llvm::isBitwiseNot(SDValue V) {
   if (V.getOpcode() != ISD::XOR)
     return false;
   ConstantSDNode *C = isConstOrConstSplat(peekThroughBitcasts(V.getOperand(1)));
   return C && C->isAllOnesValue();
 }
 
-ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs) {
+static ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs,
+                                           bool IgnoreBitWidth) {
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
     return CN;
 
   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
     BitVector UndefElements;
     ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
 
     // BuildVectors can truncate their operands. Ignore that case here.
     if (CN && (UndefElements.none() || AllowUndefs) &&
-        CN->getValueType(0) == N.getValueType().getScalarType())
+        (IgnoreBitWidth ||
+         CN->getValueType(0) == N.getValueType().getScalarType()))
       return CN;
   }
 
   return nullptr;
 }
 
+ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs) {
+  return ::isConstOrConstSplat(N, AllowUndefs, /*IgnoreBitWidth=*/false);
+}
+
 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                           bool AllowUndefs) {
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
     return CN;
 
   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
     BitVector UndefElements;
     ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
if (CN && (UndefElements.none() || AllowUndefs))
       return CN;
   }
 
   return nullptr;
 }
 
 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
   // TODO: may want to use peekThroughBitcast() here.
-  ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
+  ConstantSDNode *C =
+      ::isConstOrConstSplat(N, AllowUndefs, /*IgnoreBitWidth=*/true);
   return C && C->isNullValue();
 }
 
 bool llvm::isOneOrOneSplat(SDValue N) {
   // TODO: may want to use peekThroughBitcast() here.
   unsigned BitWidth = N.getScalarValueSizeInBits();
   ConstantSDNode *C = isConstOrConstSplat(N);
   return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth;

llvm/test/CodeGen/AArch64/vecreduce-bool.ll

; CHECK-NEXT:    ret
   %y = call i1 @llvm.experimental.vector.reduce.and.i1.v16i1(<16 x i1> %x)
   %z = select i1 %y, i32 %a1, i32 %a2
   ret i32 %z
 }
 
 define i32 @reduce_and_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-LABEL: reduce_and_v32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    uminv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    csel w0, w0, w1, ne
 ; CHECK-NEXT:    ret
   %x = icmp slt <32 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.experimental.vector.reduce.and.i1.v32i1(<32 x i1> %x)
   %z = select i1 %y, i32 %a1, i32 %a2
; CHECK-NEXT:    ret
   %y = call i1 @llvm.experimental.vector.reduce.or.i1.v16i1(<16 x i1> %x)
   %z = select i1 %y, i32 %a1, i32 %a2
   ret i32 %z
 }
 
 define i32 @reduce_or_v32(<32 x i8> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-LABEL: reduce_or_v32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    umaxv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    tst w8, #0x1
 ; CHECK-NEXT:    csel w0, w0, w1, ne
 ; CHECK-NEXT:    ret
   %x = icmp slt <32 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.experimental.vector.reduce.or.i1.v32i1(<32 x i1> %x)
   %z = select i1 %y, i32 %a1, i32 %a2
   ret i32 %z
 }

llvm/test/CodeGen/X86/avx2-masked-gather.ll

 ; NOGATHER-NEXT:    vmovapd %xmm1, %xmm0
 ; NOGATHER-NEXT:    retq
 entry:
   %ld  = load <2 x double*>, <2 x double*>* %ptr
   %res = call <2 x double> @llvm.masked.gather.v2double(<2 x double*> %ld, i32 0, <2 x i1> %masks, <2 x double> %passthro)
   ret <2 x double> %res
 }
 
+
+define <2 x double> @masked_gather_zeromask(<2 x double*>* %ptr, <2 x double> %dummy, <2 x double> %passthru) {
+; X86-LABEL: masked_gather_zeromask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vmovaps %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: masked_gather_zeromask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vmovaps %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; NOGATHER-LABEL: masked_gather_zeromask:
+; NOGATHER:       # %bb.0: # %entry
+; NOGATHER-NEXT:    vmovaps %xmm1, %xmm0
+; NOGATHER-NEXT:    retq
+entry:
+  %ld  = load <2 x double*>, <2 x double*>* %ptr
+  %res = call <2 x double> @llvm.masked.gather.v2double(<2 x double*> %ld, i32 0, <2 x i1> zeroinitializer, <2 x double> %passthru)
+  ret <2 x double> %res
+}