[Legalizer] Pass missing SDNodeFlags in split/scalarize vector results

[qiucf]

Summary

This was found when working on D76585. In some of arithmetic scalarize/split result methods, SDNodeFlags not passed down. This patch adds them back.

[spatel]

I'm not sure if this is a complete fix. But it extends what we already do, so it should be an improvement. LGTM.

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
   return DAG.getNode(N->getOpcode(), SDLoc(N),
                      LHS.getValueType(), LHS, RHS, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
   SDValue Op2 = GetScalarizedVector(N->getOperand(2));
-  return DAG.getNode(N->getOpcode(), SDLoc(N),
-                     Op0.getValueType(), Op0, Op1, Op2);
+  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
+                     Op2, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
   SDValue Op2 = N->getOperand(2);
   return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
-                     Op2);
+                     Op2, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
   EVT VT = N->getValueType(0).getVectorElementType();
   unsigned NumOpers = N->getNumOperands();
   SDValue Chain = N->getOperand(0);
   EVT ValueVTs[] = {VT, MVT::Other};
   SDLoc dl(N);
 
   SmallVector<SDValue, 4> Opers(NumOpers);
 
   // The Chain is the first operand.
   Opers[0] = Chain;
 
   // Now process the remaining operands.
   for (unsigned i = 1; i < NumOpers; ++i) {
     SDValue Oper = N->getOperand(i);
 
     if (Oper.getValueType().isVector())
       Oper = GetScalarizedVector(Oper);
 
     Opers[i] = Oper;
   }
 
-  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
+  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
+                               Opers, N->getFlags());
 
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
   ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
   return Result;
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
     ScalarLHS = ElemsLHS[0];
     ScalarRHS = ElemsRHS[0];
   }
 
   SDVTList ScalarVTs = DAG.getVTList(
       ResVT.getVectorElementType(), OvVT.getVectorElementType());
   SDNode *ScalarNode = DAG.getNode(
       N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
+  ScalarNode->setFlags(N->getFlags());
 
   // Replace the other vector result not being explicitly scalarized here.
   unsigned OtherNo = 1 - ResNo;
   EVT OtherVT = N->getValueType(OtherNo);
   if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
     SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
   } else {
     SDValue OtherVal = DAG.getNode(
SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
   // See the similar logic in ScalarizeVecRes_SETCC
   if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
     Op = GetScalarizedVector(Op);
   } else {
     EVT VT = OpVT.getVectorElementType();
     Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
                      DAG.getVectorIdxConstant(0, DL));
   }
-  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
+  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
   EVT EltVT = N->getValueType(0).getVectorElementType();
   EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
   SDValue LHS = GetScalarizedVector(N->getOperand(0));
   return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
                      LHS, DAG.getValueType(ExtVT));
void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
   SDValue Op0Lo, Op0Hi;
   GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
   SDValue Op1Lo, Op1Hi;
   GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
   SDValue Op2Lo, Op2Hi;
   GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
   SDLoc dl(N);
 
-  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
-                   Op0Lo, Op1Lo, Op2Lo);
-  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
-                   Op0Hi, Op1Hi, Op2Hi);
+  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
+                   Op2Lo, N->getFlags());
+  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
+                   Op2Hi, N->getFlags());
 }
 
 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
   SDValue LHSLo, LHSHi;
   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
   SDValue RHSLo, RHSHi;
   GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
   SDLoc dl(N);
   SDValue Op2 = N->getOperand(2);
 
   unsigned Opcode = N->getOpcode();
-  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
-  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
+  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
+                   N->getFlags());
+  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
+                   N->getFlags());
 }
 
 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
                                            SDValue &Hi) {
   // We know the result is a vector.  The input may be either a vector or a
   // scalar value.
   EVT LoVT, HiVT;
   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
for (unsigned i = 1; i < NumOps; ++i) {
     }
 
     OpsLo[i] = OpLo;
     OpsHi[i] = OpHi;
   }
 
   EVT LoValueVTs[] = {LoVT, MVT::Other};
   EVT HiValueVTs[] = {HiVT, MVT::Other};
-  Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
-  Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
+  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
+                   N->getFlags());
+  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
+                   N->getFlags());
 
   // Build a factor node to remember that this Op is independent of the
   // other one.
   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                       Lo.getValue(1), Hi.getValue(1));
 
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
     std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
   }
 
   unsigned Opcode = N->getOpcode();
   SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
   SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
   SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
   SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
+  LoNode->setFlags(N->getFlags());
+  HiNode->setFlags(N->getFlags());
 
   Lo = SDValue(LoNode, ResNo);
   Hi = SDValue(HiNode, ResNo);
 
   // Replace the other vector result not being explicitly split here.
   unsigned OtherNo = 1 - ResNo;
   EVT OtherVT = N->getValueType(OtherNo);
   if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
   unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
   EVT InVT = N->getOperand(OpNo).getValueType();
   if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
     GetSplitVector(N->getOperand(OpNo), Lo, Hi);
   else
     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
 
   if (N->getOpcode() == ISD::FP_ROUND) {
-    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
-    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
+    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
+                     N->getFlags());
+    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
+                     N->getFlags());
   } else {
-    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
-    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
+    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
+    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
   }
 }
 
 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
                                             SDValue &Hi) {
   SDLoc dl(N);
   EVT SrcVT = N->getOperand(0).getValueType();
   EVT DestVT = N->getValueType(0);

llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll

 ; SKL-NEXT:    retq
   %call = tail call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %f) #2
   ret <4 x float> %call
 }
 
 define <8 x float> @v8f32_no_daz(<8 x float> %f) #0 {
 ; NHM-LABEL: v8f32_no_daz:
 ; NHM:       # %bb.0:
-; NHM-NEXT:    sqrtps %xmm0, %xmm0
-; NHM-NEXT:    sqrtps %xmm1, %xmm1
+; NHM-NEXT:    movaps %xmm0, %xmm2
+; NHM-NEXT:    rsqrtps %xmm0, %xmm3
+; NHM-NEXT:    mulps %xmm3, %xmm0
+; NHM-NEXT:    movaps {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; NHM-NEXT:    movaps %xmm0, %xmm5
+; NHM-NEXT:    mulps %xmm4, %xmm5
+; NHM-NEXT:    mulps %xmm3, %xmm0
+; NHM-NEXT:    movaps {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; NHM-NEXT:    addps %xmm3, %xmm0
+; NHM-NEXT:    mulps %xmm5, %xmm0
+; NHM-NEXT:    movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN]
+; NHM-NEXT:    andps %xmm5, %xmm2
+; NHM-NEXT:    movaps {{.*#+}} xmm6 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]
+; NHM-NEXT:    movaps %xmm6, %xmm7
+; NHM-NEXT:    cmpleps %xmm2, %xmm7
+; NHM-NEXT:    andps %xmm7, %xmm0
+; NHM-NEXT:    rsqrtps %xmm1, %xmm7
+; NHM-NEXT:    movaps %xmm1, %xmm2
+; NHM-NEXT:    mulps %xmm7, %xmm2
+; NHM-NEXT:    mulps %xmm2, %xmm4
+; NHM-NEXT:    mulps %xmm7, %xmm2
+; NHM-NEXT:    addps %xmm3, %xmm2
+; NHM-NEXT:    mulps %xmm4, %xmm2
+; NHM-NEXT:    andps %xmm5, %xmm1
+; NHM-NEXT:    cmpleps %xmm1, %xmm6
+; NHM-NEXT:    andps %xmm6, %xmm2
+; NHM-NEXT:    movaps %xmm2, %xmm1
 ; NHM-NEXT:    retq
 ;
 ; SNB-LABEL: v8f32_no_daz:
 ; SNB:       # %bb.0:
 ; SNB-NEXT:    vrsqrtps %ymm0, %ymm1
 ; SNB-NEXT:    vmulps %ymm1, %ymm0, %ymm2
 ; SNB-NEXT:    vmulps {{.*}}(%rip), %ymm2, %ymm3
 ; SNB-NEXT:    vmulps %ymm1, %ymm2, %ymm1

 ; SKL-NEXT:    retq
   %call = tail call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %f) #2
   ret <4 x float> %call
 }
 
 define <8 x float> @v8f32_daz(<8 x float> %f) #1 {
 ; NHM-LABEL: v8f32_daz:
 ; NHM:       # %bb.0:
-; NHM-NEXT:    sqrtps %xmm0, %xmm0
-; NHM-NEXT:    sqrtps %xmm1, %xmm1
+; NHM-NEXT:    rsqrtps %xmm0, %xmm2
+; NHM-NEXT:    movaps %xmm0, %xmm3
+; NHM-NEXT:    mulps %xmm2, %xmm3
+; NHM-NEXT:    movaps {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; NHM-NEXT:    movaps %xmm3, %xmm5
+; NHM-NEXT:    mulps %xmm4, %xmm5
+; NHM-NEXT:    mulps %xmm2, %xmm3
+; NHM-NEXT:    movaps {{.*#+}} xmm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; NHM-NEXT:    addps %xmm2, %xmm3
+; NHM-NEXT:    mulps %xmm5, %xmm3
+; NHM-NEXT:    xorps %xmm5, %xmm5
+; NHM-NEXT:    cmpneqps %xmm5, %xmm0
+; NHM-NEXT:    andps %xmm3, %xmm0
+; NHM-NEXT:    rsqrtps %xmm1, %xmm3
+; NHM-NEXT:    movaps %xmm1, %xmm6
+; NHM-NEXT:    mulps %xmm3, %xmm6
+; NHM-NEXT:    mulps %xmm6, %xmm4
+; NHM-NEXT:    mulps %xmm3, %xmm6
+; NHM-NEXT:    addps %xmm2, %xmm6
+; NHM-NEXT:    mulps %xmm4, %xmm6
+; NHM-NEXT:    cmpneqps %xmm5, %xmm1
+; NHM-NEXT:    andps %xmm6, %xmm1
 ; NHM-NEXT:    retq
 ;
 ; SNB-LABEL: v8f32_daz:
 ; SNB:       # %bb.0:
 ; SNB-NEXT:    vrsqrtps %ymm0, %ymm1
 ; SNB-NEXT:    vmulps %ymm1, %ymm0, %ymm2
 ; SNB-NEXT:    vmulps {{.*}}(%rip), %ymm2, %ymm3
 ; SNB-NEXT:    vmulps %ymm1, %ymm2, %ymm1