[Power9] Ensure float128 in non-homogenous aggregates are passed via VSX registers

[lei]

Summary

Non-homogenous aggregates are passed in consecutive GPRs, in GPRs and in memory, or in memory. This patch ensures that float128 members of non-homogenous aggregates are passed via VSX registers.

This is done via custom lowering a bitcast of a build_pari(i64,i64) to float128 to a new PPCISD node, BUILD_FP128.

[nemanjai]

Other than a few cosmetic changes, LGTM.

lib/Target/PowerPC/PPCISelLowering.cpp

7662

A bitcast produces a single result, why getValueType(0) vs. just getValueType()?

7668

There are 3 more uses of this for the if statement above, why not define it up there and use it in the condition as well?

7669

No need for this temp. Just return the result of getNode().

[lei]

lib/Target/PowerPC/PPCISelLowering.cpp

7662

Op.getValueType() == Op->getValueType(0).
I can update to use Op.getValueType()

[nemanjai]

lib/Target/PowerPC/PPCISelLowering.cpp

7662

Correct, the two are equivalent. But the former calls a member function of SDValue directly while the latter calls operator->() and calls a member function of the containing SDNode.

lib/Target/PowerPC/PPCISelLowering.h

enum NodeType : unsigned {
       MFVSR,
 
       /// Direct move from a GPR to a VSX register (algebraic)
       MTVSRA,
 
       /// Direct move from a GPR to a VSX register (zero)
       MTVSRZ,
 
+      /// Direct move of 2 consective GPR to a VSX register.
+      BUILD_FP128,
+
       /// Extract a subvector from signed integer vector and convert to FP.
       /// It is primarily used to convert a (widened) illegal integer vector
       /// type to a legal floating point vector type.
       /// For example v2i32 -> widened to v4i32 -> v2f64
       SINT_VEC_TO_FP,
 
       /// Extract a subvector from unsigned integer vector and convert to FP.
       /// As with SINT_VEC_TO_FP, used for converting illegal types.
SDValue LowerCall_32SVR4(SDValue Chain, SDValue Callee,
                              const SmallVectorImpl<SDValue> &OutVals,
                              const SmallVectorImpl<ISD::InputArg> &Ins,
                              const SDLoc &dl, SelectionDAG &DAG,
                              SmallVectorImpl<SDValue> &InVals,
                              ImmutableCallSite CS) const;
 
     SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
 
     SDValue DAGCombineExtBoolTrunc(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue combineStoreFPToInt(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue combineSHL(SDNode *N, DAGCombinerInfo &DCI) const;
     SDValue combineSRA(SDNode *N, DAGCombinerInfo &DCI) const;

lib/Target/PowerPC/PPCISelLowering.cpp

if (Subtarget.hasP9Vector()) {
         // No extending loads to f128 on PPC.
         for (MVT FPT : MVT::fp_valuetypes())
           setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand);
         setOperationAction(ISD::FMA, MVT::f128, Legal);
         setOperationAction(ISD::FP_ROUND, MVT::f64, Legal);
         setOperationAction(ISD::FP_ROUND, MVT::f32, Legal);
         setTruncStoreAction(MVT::f128, MVT::f64, Expand);
         setTruncStoreAction(MVT::f128, MVT::f32, Expand);
+        setOperationAction(ISD::BITCAST, MVT::i128, Custom);
       }
 
     }
 
     if (Subtarget.hasP9Altivec()) {
       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
     }
const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::XXSWAPD:         return "PPCISD::XXSWAPD";
   case PPCISD::SWAP_NO_CHAIN:   return "PPCISD::SWAP_NO_CHAIN";
   case PPCISD::QVFPERM:         return "PPCISD::QVFPERM";
   case PPCISD::QVGPCI:          return "PPCISD::QVGPCI";
   case PPCISD::QVALIGNI:        return "PPCISD::QVALIGNI";
   case PPCISD::QVESPLATI:       return "PPCISD::QVESPLATI";
   case PPCISD::QBFLT:           return "PPCISD::QBFLT";
   case PPCISD::QVLFSb:          return "PPCISD::QVLFSb";
+  case PPCISD::BUILD_FP128:     return "PPCISD::BUILD_FP128";
   }
   return nullptr;
 }
 
 EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
                                           EVT VT) const {
   if (!VT.isVector())
     return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
for (int i = 0, e = V->getNumOperands(); i < e; ++i) {
     // uses than just this BV node, then it isn't a splat.
     if (V->getOperand(i) != Op0 ||
         (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode())))
       IsSplat = false;
   }
   return !(IsSplat && IsLoad);
 }
 
+// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128.
+SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
+
+  SDLoc dl(Op);
+
+  if (!EnableQuadPrecision ||
+      (Op->getValueType(0) != MVT::f128 ) ||
+      (Op->getOperand(0).getOpcode() != ISD::BUILD_PAIR) ||
+      (Op->getOperand(0).getOperand(0).getValueType() !=  MVT::i64) ||
+      (Op->getOperand(0).getOperand(1).getValueType() != MVT::i64))
+    return SDValue();
+
+  SDValue Op0 = Op->getOperand(0);
+  SDValue Val = DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128,
+                            Op0.getOperand(0), Op0.getOperand(1));
+
+  return Val;
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.  If we CAN select this case, and if it
 // selects to a single instruction, return Op.  Otherwise, if we can codegen
 // this case more efficiently than a constant pool load, lower it to the
 // sequence of ops that should be used.
 SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                              SelectionDAG &DAG) const {
   SDLoc dl(Op);
SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SIGN_EXTEND_INREG:  return LowerSIGN_EXTEND_INREG(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::MUL:                return LowerMUL(Op, DAG);
 
   // For counter-based loop handling.
   case ISD::INTRINSIC_W_CHAIN:  return SDValue();
 
+  case ISD::BITCAST:            return LowerBITCAST(Op, DAG);
+
   // Frame & Return address.
   case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
   case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
 
   case ISD::INTRINSIC_VOID:
     return LowerINTRINSIC_VOID(Op, DAG);
   case ISD::SREM:
   case ISD::UREM:

lib/Target/PowerPC/PPCInstrInfo.td

 def PPCcmpb     : SDNode<"PPCISD::CMPB", SDTIntBinOp, []>;
 
 // These nodes represent the 32-bit PPC shifts that operate on 6-bit shift
 // amounts.  These nodes are generated by the multi-precision shift code.
 def PPCsrl        : SDNode<"PPCISD::SRL"       , SDTIntShiftOp>;
 def PPCsra        : SDNode<"PPCISD::SRA"       , SDTIntShiftOp>;
 def PPCshl        : SDNode<"PPCISD::SHL"       , SDTIntShiftOp>;
 
+// Move 2 i64 values into a VSX register
+def PPCbuild_fp128: SDNode<"PPCISD::BUILD_FP128",
+                           SDTypeProfile<1, 2,
+                             [SDTCisFP<0>, SDTCisSameSizeAs<1,2>,
+                              SDTCisSameAs<1,2>]>,
+                           []>;
+
 // These are target-independent nodes, but have target-specific formats.
 def callseq_start : SDNode<"ISD::CALLSEQ_START", SDT_PPCCallSeqStart,
                            [SDNPHasChain, SDNPOutGlue]>;
 def callseq_end   : SDNode<"ISD::CALLSEQ_END",   SDT_PPCCallSeqEnd,
                            [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
 def SDT_PPCCall   : SDTypeProfile<0, -1, [SDTCisInt<0>]>;
 def PPCcall  : SDNode<"PPCISD::CALL", SDT_PPCCall,

lib/Target/PowerPC/PPCInstrVSX.td

def : Pat<(PPCstore_scal_int_from_vsr
               (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), xoaddr:$dst, 1),
             (STXSIBX (XSCVDPUXWS f64:$src), xoaddr:$dst)>;
 
   // Round & Convert QP -> DP/SP
   def : Pat<(f64 (fpround f128:$src)), (f64 (XSCVQPDP $src))>;
   def : Pat<(f32 (fpround f128:$src)), (f32 (XSRSP (XSCVQPDPO $src)))>;
 } // end HasP9Vector, AddedComplexity
 
+let AddedComplexity = 400 in {
+  let Predicates = [IsISA3_0, HasP9Vector, HasDirectMove, IsBigEndian] in {
+    def : Pat<(f128 (PPCbuild_fp128 i64:$rB, i64:$rA)),
+              (f128 (COPY_TO_REGCLASS (MTVSRDD $rB, $rA), VRRC))>;
+  }
+  let Predicates = [IsISA3_0, HasP9Vector, HasDirectMove, IsLittleEndian] in {
+    def : Pat<(f128 (PPCbuild_fp128 i64:$rA, i64:$rB)),
+              (f128 (COPY_TO_REGCLASS (MTVSRDD $rB, $rA), VRRC))>;
+  }
+}
+
 let Predicates = [HasP9Vector] in {
   let isPseudo = 1 in {
     let mayStore = 1 in {
       def SPILLTOVSR_STX : PseudoXFormMemOp<(outs),
                                             (ins spilltovsrrc:$XT, memrr:$dst),
                                             "#SPILLTOVSR_STX", []>;
       def SPILLTOVSR_ST : Pseudo<(outs), (ins spilltovsrrc:$XT, memrix:$dst),
                                 "#SPILLTOVSR_ST", []>;

test/CodeGen/PowerPC/f128-aggregates.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -enable-ppc-quad-precision -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64-unknown-unknown \
+; RUN:   -enable-ppc-quad-precision -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK-BE %s
+
+%struct.MixedC = type { i32, %struct.SA, float, [12 x i8] }
+%struct.SA = type { double, fp128, <4 x float> }
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @testMixedAggregate([3 x i128] %a.coerce) {
+; CHECK-LABEL: testMixedAggregate:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtvsrdd 34, 8, 7
+; CHECK-NEXT:    blr
+
+; CHECK-BE-LABEL: testMixedAggregate:
+; CHECK-BE:       mtvsrdd 34, 8, 7
+; CHECK-BE-NEXT:  blr
+entry:
+  %a.coerce.fca.2.extract = extractvalue [3 x i128] %a.coerce, 2
+  %0 = bitcast i128 %a.coerce.fca.2.extract to fp128
+  ret fp128 %0
+}
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @testMixedAggregate_02([4 x i128] %a.coerce) {
+; CHECK-LABEL: testMixedAggregate_02:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtvsrdd 34, 6, 5
+; CHECK-NEXT:    blr
+
+; CHECK-BE-LABEL: testMixedAggregate_02:
+; CHECK-BE:       mtvsrdd 34, 6, 5
+; CHECK-BE-NEXT:  blr
+entry:
+  %a.coerce.fca.1.extract = extractvalue [4 x i128] %a.coerce, 1
+  %0 = bitcast i128 %a.coerce.fca.1.extract to fp128
+  ret fp128 %0
+}
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @testMixedAggregate_03([4 x i128] %sa.coerce) {
+; CHECK-LABEL: testMixedAggregate_03:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtvsrwa 34, 3
+; CHECK-NEXT:    mtvsrdd 35, 6, 5
+; CHECK-NEXT:    mtvsrd 36, 10
+; CHECK-NEXT:    xscvsdqp 2, 2
+; CHECK-NEXT:    xscvsdqp 19, 4
+; CHECK-NEXT:    xsaddqp 2, 3, 2
+; CHECK-NEXT:    xsaddqp 2, 2, 19
+; CHECK-NEXT:    blr
+entry:
+  %sa.coerce.fca.0.extract = extractvalue [4 x i128] %sa.coerce, 0
+  %sa.sroa.0.0.extract.trunc = trunc i128 %sa.coerce.fca.0.extract to i32
+  %sa.coerce.fca.1.extract = extractvalue [4 x i128] %sa.coerce, 1
+  %sa.coerce.fca.3.extract = extractvalue [4 x i128] %sa.coerce, 3
+  %sa.sroa.6.48.extract.shift = lshr i128 %sa.coerce.fca.3.extract, 64
+  %sa.sroa.6.48.extract.trunc = trunc i128 %sa.sroa.6.48.extract.shift to i64
+  %conv = sitofp i32 %sa.sroa.0.0.extract.trunc to fp128
+  %0 = bitcast i128 %sa.coerce.fca.1.extract to fp128
+  %add = fadd fp128 %0, %conv
+  %conv2 = sitofp i64 %sa.sroa.6.48.extract.trunc to fp128
+  %add3 = fadd fp128 %add, %conv2
+  ret fp128 %add3
+}
+
+
+; Function Attrs: norecurse nounwind readonly
+define fp128 @testNestedAggregate(%struct.MixedC* byval nocapture readonly align 16 %a) {
+; CHECK-LABEL: testNestedAggregate:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    std 8, 72(1)
+; CHECK-NEXT:    std 7, 64(1)
+; CHECK-NEXT:    std 10, 88(1)
+; CHECK-NEXT:    std 9, 80(1)
+; CHECK-NEXT:    lxv 34, 64(1)
+; CHECK-NEXT:    std 6, 56(1)
+; CHECK-NEXT:    std 5, 48(1)
+; CHECK-NEXT:    std 4, 40(1)
+; CHECK-NEXT:    std 3, 32(1)
+; CHECK-NEXT:    blr
+
+; CHECK-BE-LABEL: testNestedAggregate:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    std 8, 88(1)
+; CHECK-BE-NEXT:    std 7, 80(1)
+; CHECK-BE-NEXT:    std 10, 104(1)
+; CHECK-BE-NEXT:    std 9, 96(1)
+; CHECK-BE-NEXT:    lxv 34, 80(1)
+; CHECK-BE-NEXT:    std 6, 72(1)
+; CHECK-BE-NEXT:    std 5, 64(1)
+; CHECK-BE-NEXT:    std 4, 56(1)
+; CHECK-BE-NEXT:    std 3, 48(1)
+; CHECK-BE-NEXT:    blr
+entry:
+  %c = getelementptr inbounds %struct.MixedC, %struct.MixedC* %a, i64 0, i32 1, i32 1
+  %0 = load fp128, fp128* %c, align 16
+  ret fp128 %0
+}
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @testUnion_01([1 x i128] %a.coerce) {
+; CHECK-LABEL: testUnion_01:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtvsrdd 34, 4, 3
+; CHECK-NEXT:    blr
+
+; CHECK-BE-LABEL: testUnion_01:
+; CHECK-BE:       mtvsrdd 34, 4, 3
+; CHECK-BE-NEXT:  blr
+entry:
+  %a.coerce.fca.0.extract = extractvalue [1 x i128] %a.coerce, 0
+  %0 = bitcast i128 %a.coerce.fca.0.extract to fp128
+  ret fp128 %0
+}
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @testUnion_02([1 x i128] %a.coerce) {
+; CHECK-LABEL: testUnion_02:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtvsrdd 34, 4, 3
+; CHECK-NEXT:    blr
+
+; CHECK-BE-LABEL: testUnion_02:
+; CHECK-BE:       mtvsrdd 34, 4, 3
+; CHECK-BE-NEXT:  blr
+entry:
+  %a.coerce.fca.0.extract = extractvalue [1 x i128] %a.coerce, 0
+  %0 = bitcast i128 %a.coerce.fca.0.extract to fp128
+  ret fp128 %0
+}
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @testUnion_03([4 x i128] %a.coerce) {
+; CHECK-LABEL: testUnion_03:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mtvsrdd 34, 8, 7
+; CHECK-NEXT:    blr
+
+; CHECK-BE-LABEL: testUnion_03:
+; CHECK-BE:       mtvsrdd 34, 8, 7
+; CHECK-BE-NEXT:  blr
+entry:
+  %a.coerce.fca.2.extract = extractvalue [4 x i128] %a.coerce, 2
+  %0 = bitcast i128 %a.coerce.fca.2.extract to fp128
+  ret fp128 %0
+}
+
+; Function Attrs: nounwind
+define fp128 @sum_float128(i32 signext %count, ...) {
+; CHECK-LABEL: sum_float128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis 11, 2, .LCPI7_0@toc@ha
+; CHECK-NEXT:    cmpwi 0, 3, 1
+; CHECK-NEXT:    std 10, 88(1)
+; CHECK-NEXT:    std 9, 80(1)
+; CHECK-NEXT:    std 8, 72(1)
+; CHECK-NEXT:    std 7, 64(1)
+; CHECK-NEXT:    std 6, 56(1)
+; CHECK-NEXT:    std 5, 48(1)
+; CHECK-NEXT:    std 4, 40(1)
+; CHECK-NEXT:    addi 11, 11, .LCPI7_0@toc@l
+; CHECK-NEXT:    lxvx 34, 0, 11
+; CHECK-NEXT:    bltlr 0
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    addi 3, 1, 40
+; CHECK-NEXT:    lxvx 35, 0, 3
+; CHECK-NEXT:    lxv 51, 16(3)
+; CHECK-NEXT:    addi 3, 1, 72
+; CHECK-NEXT:    std 3, -8(1)
+; CHECK-NEXT:    xsaddqp 2, 3, 2
+; CHECK-NEXT:    xsaddqp 2, 2, 19
+; CHECK-NEXT:    blr
+entry:
+  %ap = alloca i8*, align 8
+  %0 = bitcast i8** %ap to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %0) #2
+  %cmp = icmp slt i32 %count, 1
+  br i1 %cmp, label %cleanup, label %if.end
+
+if.end:                                           ; preds = %entry
+  call void @llvm.va_start(i8* nonnull %0)
+  %argp.cur = load i8*, i8** %ap, align 8
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i64 16
+  %1 = bitcast i8* %argp.cur to fp128*
+  %2 = load fp128, fp128* %1, align 8
+  %add = fadd fp128 %2, 0xL00000000000000000000000000000000
+  %argp.next3 = getelementptr inbounds i8, i8* %argp.cur, i64 32
+  store i8* %argp.next3, i8** %ap, align 8
+  %3 = bitcast i8* %argp.next to fp128*
+  %4 = load fp128, fp128* %3, align 8
+  %add4 = fadd fp128 %add, %4
+  call void @llvm.va_end(i8* nonnull %0)
+  br label %cleanup
+
+cleanup:                                          ; preds = %entry, %if.end
+  %retval.0 = phi fp128 [ %add4, %if.end ], [ 0xL00000000000000000000000000000000, %entry ]
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %0) #2
+  ret fp128 %retval.0
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+declare void @llvm.va_start(i8*) #2
+declare void @llvm.va_end(i8*) #2
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1