Index: include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- include/llvm/CodeGen/ISDOpcodes.h +++ include/llvm/CodeGen/ISDOpcodes.h @@ -256,6 +256,14 @@ /// Same for multiplication. SMULO, UMULO, + /// RESULT = SADDSAT(LHS, RHS) - Perform signed saturation addition on 2 + /// integers with the same bit width (W). If the true value of LHS + RHS + /// exceeds the largest signed value that can be represented by W bits, the + /// resulting value is this maximum value. Otherwise, if this value is less + /// than the smallest signed value that can be represented by W bits, the + /// resulting value is this minimum value. + SADDSAT, + /// Simple binary floating point operators. FADD, FSUB, FMUL, FDIV, FREM, Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -797,6 +797,11 @@ return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op]; } + /// Method for building the DAG expansion of ISD::SADDSAT. This method accepts + /// integers or vectors of integers as its arguments. + SDValue getExpandedSignedSaturationAddition(SDNode *Node, + SelectionDAG &DAG) const; + LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const { unsigned EqOpc; switch (Op) { @@ -2573,6 +2578,12 @@ /// Set default libcall names and calling conventions. void InitLibcalls(const Triple &TT); + /// Method for building the DAG expansion of ISD::SADDSAT. This method accepts + /// only integer arguments. + SDValue getExpandedIntegerSignedSaturationAddition(SDValue LHS, SDValue RHS, + SDLoc dl, + SelectionDAG &DAG) const; + protected: /// Return true if the extension represented by \p I is free. /// \pre \p I is a sign, zero, or fp extension and Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -700,6 +700,12 @@ [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; +//===------------------------- Fixed Point Intrinsics ---------------------===// +// +def int_sadd_sat : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem, IntrSpeculatable, Commutative]>; + //===------------------------- Memory Use Markers -------------------------===// // def int_lifetime_start : Intrinsic<[], Index: include/llvm/Target/TargetSelectionDAG.td =================================================================== --- include/llvm/Target/TargetSelectionDAG.td +++ include/llvm/Target/TargetSelectionDAG.td @@ -373,6 +373,8 @@ def umax : SDNode<"ISD::UMAX" , SDTIntBinOp, [SDNPCommutative, SDNPAssociative]>; +def saddsat : SDNode<"ISD::SADDSAT" , SDTIntBinOp, [SDNPCommutative]>; + def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>; def sext_invec : SDNode<"ISD::SIGN_EXTEND_VECTOR_INREG", SDTExtInvec>; def zext_invec : SDNode<"ISD::ZERO_EXTEND_VECTOR_INREG", SDTExtInvec>; Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1115,6 +1115,10 @@ Action = TLI.getStrictFPOperationAction(Node->getOpcode(), Node->getValueType(0)); break; + case ISD::SADDSAT: { + Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); + break; + } case ISD::MSCATTER: Action = TLI.getOperationAction(Node->getOpcode(), cast(Node)->getValue().getValueType()); @@ -3460,6 +3464,10 @@ } break; } + case ISD::SADDSAT: { + Results.push_back(TLI.getExpandedSignedSaturationAddition(Node, DAG)); + break; + } case ISD::SADDO: case ISD::SSUBO: { SDValue LHS = Node->getOperand(0); Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -141,6 +141,8 @@ case ISD::ADDCARRY: case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; + case ISD::SADDSAT: Res = PromoteIntRes_SADDSAT(N); break; + case ISD::ATOMIC_LOAD: Res = PromoteIntRes_Atomic0(cast(N)); break; @@ -534,6 +536,35 @@ return SDValue(Res.getNode(), 1); } +SDValue DAGTypeLegalizer::PromoteIntRes_SADDSAT(SDNode *N) { + // For promoting iN -> iM, this can be expanded by + // 1. ANY_EXTEND iN to iM + // 2. SHL by M-N + // 3. SADDSAT + // 4. ASHR by M-N + SDLoc dl(N); + SDValue Op1 = N->getOperand(0); + SDValue Op2 = N->getOperand(1); + unsigned OldBits = Op1.getValueSizeInBits(); + + SDValue Op1Promoted = GetPromotedInteger(Op1); + SDValue Op2Promoted = GetPromotedInteger(Op2); + + EVT PromotedType = Op1Promoted.getValueType(); + unsigned NewBits = Op1Promoted.getValueSizeInBits(); + unsigned SHLAmount = NewBits - OldBits; + EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); + SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT); + Op1Promoted = + DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); + Op2Promoted = + DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); + + SDValue Result = + DAG.getNode(ISD::SADDSAT, dl, PromotedType, Op1Promoted, Op2Promoted); + return DAG.getNode(ISD::SRA, dl, PromotedType, Result, ShiftAmount); +} + SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { if (ResNo == 1) return PromoteIntRes_Overflow(N); @@ -1454,6 +1485,8 @@ case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; case ISD::UMULO: case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; + + case ISD::SADDSAT: ExpandIntRes_SADDSAT(N, Lo, Hi); break; } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -2416,6 +2449,12 @@ ReplaceValueWith(SDValue(N, 1), R.getValue(2)); } +void DAGTypeLegalizer::ExpandIntRes_SADDSAT(SDNode *N, SDValue &Lo, + SDValue &Hi) { + SDValue Result = TLI.getExpandedSignedSaturationAddition(N, DAG); + SplitInteger(Result, Lo, Hi); +} + void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, SDValue &Lo, SDValue &Hi) { SDValue LHS = Node->getOperand(0); Index: lib/CodeGen/SelectionDAG/LegalizeTypes.h =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -330,6 +330,7 @@ SDValue PromoteIntRes_UNDEF(SDNode *N); SDValue PromoteIntRes_VAARG(SDNode *N); SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo); + SDValue PromoteIntRes_SADDSAT(SDNode *N); // Integer Operand Promotion. bool PromoteIntegerOperand(SDNode *N, unsigned OpNo); @@ -414,6 +415,7 @@ void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_SADDSAT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi); Index: lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -120,6 +120,8 @@ case ISD::UMIN: case ISD::UMAX: + case ISD::SADDSAT: + case ISD::FPOW: case ISD::FREM: case ISD::FSUB: @@ -800,6 +802,7 @@ case ISD::SMAX: case ISD::UMIN: case ISD::UMAX: + case ISD::SADDSAT: SplitVecRes_BinOp(N, Lo, Hi); break; case ISD::FMA: Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5759,6 +5759,12 @@ setValue(&I, DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Or)); return nullptr; } + case Intrinsic::sadd_sat: { + SDValue Op1 = getValue(I.getArgOperand(0)); + SDValue Op2 = getValue(I.getArgOperand(1)); + setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); + return nullptr; + } case Intrinsic::stacksave: { SDValue Op = getRoot(); Res = DAG.getNode( Index: lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -282,6 +282,8 @@ case ISD::SRA_PARTS: return "sra_parts"; case ISD::SRL_PARTS: return "srl_parts"; + case ISD::SADDSAT: return "saddsat"; + // Conversion operators. case ISD::SIGN_EXTEND: return "sign_extend"; case ISD::ZERO_EXTEND: return "zero_extend"; Index: lib/CodeGen/TargetLoweringBase.cpp =================================================================== --- lib/CodeGen/TargetLoweringBase.cpp +++ lib/CodeGen/TargetLoweringBase.cpp @@ -608,6 +608,7 @@ setOperationAction(ISD::UMIN, VT, Expand); setOperationAction(ISD::UMAX, VT, Expand); setOperationAction(ISD::ABS, VT, Expand); + setOperationAction(ISD::SADDSAT, VT, Expand); // Overflow operations default to expand setOperationAction(ISD::SADDO, VT, Expand); @@ -1857,3 +1858,73 @@ void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const { MF.getRegInfo().freezeReservedRegs(MF); } + +SDValue TargetLoweringBase::getExpandedIntegerSignedSaturationAddition( + SDValue LHS, SDValue RHS, SDLoc dl, SelectionDAG &DAG) const { + assert(LHS.getValueType().isScalarInteger() && + "Expected operands to be scalar integers"); + assert(RHS.getValueType().isScalarInteger() && + "Expected operands to be scalar integers"); + + unsigned BitWidth = LHS.getValueSizeInBits(); + EVT ResultType = LHS.getValueType(); + EVT BoolVT = + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ResultType); + SDValue Result = + DAG.getNode(ISD::SADDO, dl, DAG.getVTList(ResultType, BoolVT), LHS, RHS); + SDValue Sum = Result.getValue(0); + SDValue Overflow = Result.getValue(1); + + // SatMax -> Overflow && Sum < 0 + // SatMin -> Overflow && Sum > 0 + SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); + + SDValue SumPos = DAG.getSetCC(dl, BoolVT, Sum, Zero, ISD::SETGE); + SDValue SumNeg = DAG.getSetCC(dl, BoolVT, Sum, Zero, ISD::SETLT); + SDValue UseSatMax = DAG.getNode(ISD::AND, dl, BoolVT, Overflow, SumNeg); + SDValue UseSatMin = DAG.getNode(ISD::AND, dl, BoolVT, Overflow, SumPos); + + APInt MinVal = APInt::getSignedMinValue(BitWidth); + APInt MaxVal = APInt::getSignedMaxValue(BitWidth); + SDValue SatMin = DAG.getConstant(MinVal, dl, ResultType); + SDValue SatMax = DAG.getConstant(MaxVal, dl, ResultType); + + Result = DAG.getSelect(dl, ResultType, UseSatMax, SatMax, Sum); + return DAG.getSelect(dl, ResultType, UseSatMin, SatMin, Result); +} + +SDValue TargetLoweringBase::getExpandedSignedSaturationAddition( + SDNode *Node, SelectionDAG &DAG) const { + assert(Node->getOpcode() == ISD::SADDSAT && + "Expected method to receive SADDSAT node."); + assert(Node->getNumOperands() == 2 && + "Expected SADDSAT node to have 2 operands."); + + SDLoc dl(Node); + SDValue LHS = Node->getOperand(0); + SDValue RHS = Node->getOperand(1); + assert(LHS.getValueType().isInteger() && + "Expected operands to be integers or vectors of integers in SADDSAT"); + assert(RHS.getValueType().isInteger() && + "Expected operands to be integers or vectors of integers in SADDSAT"); + assert(LHS.getValueType() == RHS.getValueType() && + "Expected both operands of SADDSAT to be the same type"); + + if (LHS.getValueType().isScalarInteger()) + return getExpandedIntegerSignedSaturationAddition(LHS, RHS, dl, DAG); + + EVT ResultType = LHS.getValueType(); + EVT ElemType = ResultType.getVectorElementType(); + unsigned NumElts = ResultType.getVectorNumElements(); + SmallVector Ops(NumElts); + for (unsigned i = 0; i < NumElts; ++i) { + SDValue Idx = DAG.getConstant(i, dl, MVT::i32); + SDValue LHSElem = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemType, LHS, Idx); + SDValue RHSElem = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemType, RHS, Idx); + Ops[i] = + getExpandedIntegerSignedSaturationAddition(LHSElem, RHSElem, dl, DAG); + } + return DAG.getBuildVector(ResultType, dl, Ops); +} Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -4474,6 +4474,15 @@ break; } + case Intrinsic::sadd_sat: { + Value *Op1 = CS.getArgOperand(0); + Value *Op2 = CS.getArgOperand(1); + Assert(Op1->getType()->isIntOrIntVectorTy(), + "first operand of sadd_sat must be an int type or vector of ints"); + Assert(Op2->getType()->isIntOrIntVectorTy(), + "second operand of sadd_sat must be an int type or vector of ints"); + break; + } }; } Index: test/CodeGen/X86/sadd_sat.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/sadd_sat.ll @@ -0,0 +1,177 @@ +; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s +; RUN: llc < %s -mcpu=generic -march=x86 | FileCheck %s --check-prefix=CHECK32 + +declare i4 @llvm.sadd.sat.i4 (i4, i4) +declare i32 @llvm.sadd.sat.i32 (i32, i32) +declare i64 @llvm.sadd.sat.i64 (i64, i64) +declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>) + +define i32 @func(i32 %x, i32 %y) { +; CHECK: func +; CHECK: # %bb.0: +; CHECK-NEXT: addl %esi, %edi +; CHECK-NEXT: seto %al +; CHECK-NEXT: testl %edi, %edi +; CHECK-NEXT: setge %cl +; CHECK-NEXT: sets %dl +; CHECK-NEXT: testb %dl, %al +; CHECK-NEXT: movl $2147483647, %edx # imm = 0x7FFFFFFF +; CHECK-NEXT: cmovel %edi, %edx +; CHECK-NEXT: testb %cl, %al +; CHECK-NEXT: movl $-2147483648, %eax # imm = 0x80000000 +; CHECK-NEXT: cmovel %edx, %eax +; CHECK-NEXT: retq + %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %y); + ret i32 %tmp; +} + +define i64 @func2(i64 %x, i64 %y) { +; CHECK32: func2 +; CHECK32: # %bb.0: +; CHECK32: pushl %ebx +; CHECK32: pushl %edi +; CHECK32: pushl %esi +; CHECK32: pushl %eax +; CHECK32: movl 32(%esp), %esi +; CHECK32: movl 20(%esp), %eax +; CHECK32: movl 24(%esp), %edi +; CHECK32: addl 28(%esp), %eax +; CHECK32: movl %edi, %edx +; CHECK32: adcl %esi, %edx +; CHECK32: sets 3(%esp) # 1-byte Folded Spill +; CHECK32: setns %cl +; CHECK32: testl %edi, %edi +; CHECK32: setns %ch +; CHECK32: cmpb %cl, %ch +; CHECK32: setne %bh +; CHECK32: testl %esi, %esi +; CHECK32: setns %bl +; CHECK32: cmpb %bl, %ch +; CHECK32: sete %ch +; CHECK32: andb %bh, %ch +; CHECK32: testb %ch, 3(%esp) # 1-byte Folded Reload +; CHECK32: movl $2147483647, %esi # imm = 0x7FFFFFFF +; CHECK32: jne .LBB1_2 +; CHECK32: # %bb.1: +; CHECK32: movl %edx, %esi +; CHECK32: .LBB1_2: +; CHECK32: movl $-1, %edx +; CHECK32: jne .LBB1_4 +; CHECK32: # %bb.3: +; CHECK32: movl %eax, %edx +; CHECK32: .LBB1_4: +; CHECK32: xorl %eax, %eax +; CHECK32: testb %cl, %ch +; CHECK32: jne .LBB1_6 +; CHECK32: # %bb.5: +; CHECK32: movl %edx, %eax +; CHECK32: .LBB1_6: +; CHECK32: movl $-2147483648, %edx # imm = 0x80000000 +; CHECK32: jne .LBB1_8 +; CHECK32: # %bb.7: +; CHECK32: movl %esi, %edx +; CHECK32: .LBB1_8: +; CHECK32: addl $4, %esp +; CHECK32: popl %esi +; CHECK32: popl %edi +; CHECK32: popl %ebx +; CHECK32: retl + %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y); + ret i64 %tmp; +} + +define i4 @func3(i4 %x, i4 %y) { +; CHECK: func3 +; CHECK: # %bb.0: +; CHECK-NEXT: shlb $4, %sil +; CHECK-NEXT: shlb $4, %dil +; CHECK-NEXT: addb %sil, %dil +; CHECK-NEXT: seto %al +; CHECK-NEXT: testb %dil, %dil +; CHECK-NEXT: setge %dl +; CHECK-NEXT: sets %cl +; CHECK-NEXT: testb %cl, %al +; CHECK-NEXT: movb $127, %cl +; CHECK-NEXT: jne .LBB2_2 +; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: movl %edi, %ecx +; CHECK-NEXT: .LBB2_2: +; CHECK-NEXT: testb %dl, %al +; CHECK-NEXT: movb $-128, %al +; CHECK-NEXT: jne .LBB2_4 +; CHECK-NEXT: # %bb.3: +; CHECK-NEXT: movl %ecx, %eax +; CHECK-NEXT: .LBB2_4: +; CHECK-NEXT: sarb $4, %al +; CHECK-NEXT: retq + %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y); + ret i4 %tmp; +} + +define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) { +; CHECK: vec +; CHECK: # %bb.0: +; CHECK: pshufd $231, %xmm1, %xmm2 # xmm2 = xmm1[3,1,2,3] +; CHECK: movd %xmm2, %eax +; CHECK: pshufd $231, %xmm0, %xmm2 # xmm2 = xmm0[3,1,2,3] +; CHECK: movd %xmm2, %edx +; CHECK: addl %eax, %edx +; CHECK: seto %cl +; CHECK: testl %edx, %edx +; CHECK: setge %dil +; CHECK: sets %al +; CHECK: testb %al, %cl +; CHECK: movl $2147483647, %esi # imm = 0x7FFFFFFF +; CHECK: cmovnel %esi, %edx +; CHECK: testb %dil, %cl +; CHECK: movl $-2147483648, %edi # imm = 0x80000000 +; CHECK: cmovnel %edi, %edx +; CHECK: movd %edx, %xmm2 +; CHECK: pshufd $78, %xmm1, %xmm3 # xmm3 = xmm1[2,3,0,1] +; CHECK: movd %xmm3, %eax +; CHECK: pshufd $78, %xmm0, %xmm3 # xmm3 = xmm0[2,3,0,1] +; CHECK: movd %xmm3, %edx +; CHECK: addl %eax, %edx +; CHECK: seto %al +; CHECK: testl %edx, %edx +; CHECK: setge %r8b +; CHECK: sets %cl +; CHECK: testb %cl, %al +; CHECK: cmovnel %esi, %edx +; CHECK: testb %r8b, %al +; CHECK: cmovnel %edi, %edx +; CHECK: movd %edx, %xmm3 +; CHECK: punpckldq %xmm2, %xmm3 # xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; CHECK: movd %xmm1, %eax +; CHECK: movd %xmm0, %ecx +; CHECK: addl %eax, %ecx +; CHECK: seto %al +; CHECK: testl %ecx, %ecx +; CHECK: setge %r8b +; CHECK: sets %dl +; CHECK: testb %dl, %al +; CHECK: cmovnel %esi, %ecx +; CHECK: testb %r8b, %al +; CHECK: cmovnel %edi, %ecx +; CHECK: movd %ecx, %xmm2 +; CHECK: pshufd $229, %xmm1, %xmm1 # xmm1 = xmm1[1,1,2,3] +; CHECK: movd %xmm1, %eax +; CHECK: pshufd $229, %xmm0, %xmm0 # xmm0 = xmm0[1,1,2,3] +; CHECK: movd %xmm0, %ecx +; CHECK: addl %eax, %ecx +; CHECK: seto %al +; CHECK: testl %ecx, %ecx +; CHECK: setge %r8b +; CHECK: sets %dl +; CHECK: testb %dl, %al +; CHECK: cmovnel %esi, %ecx +; CHECK: testb %r8b, %al +; CHECK: cmovnel %edi, %ecx +; CHECK: movd %ecx, %xmm0 +; CHECK: punpckldq %xmm0, %xmm2 # xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; CHECK: punpcklqdq %xmm3, %xmm2 # xmm2 = xmm2[0],xmm3[0] +; CHECK: movdqa %xmm2, %xmm0 +; CHECK: retq + %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y); + ret <4 x i32> %tmp; +}