Index: llvm/lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -659,7 +659,6 @@ SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerVectorAND(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const; Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -549,6 +549,8 @@ // We combine OR nodes for bitfield operations. setTargetDAGCombine(ISD::OR); + // Try to create BICs for vector ANDs. + setTargetDAGCombine(ISD::AND); // Vector add and sub nodes may conceal a high-half opportunity. // Also, try to fold ADD into CSINC/CSINV.. @@ -799,7 +801,6 @@ setOperationAction(ISD::SRA, VT, Custom); setOperationAction(ISD::SRL, VT, Custom); setOperationAction(ISD::SHL, VT, Custom); - setOperationAction(ISD::AND, VT, Custom); setOperationAction(ISD::OR, VT, Custom); setOperationAction(ISD::SETCC, VT, Custom); setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); @@ -2940,8 +2941,6 @@ return LowerCTPOP(Op, DAG); case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); - case ISD::AND: - return LowerVectorAND(Op, DAG); case ISD::OR: return LowerVectorOR(Op, DAG); case ISD::XOR: @@ -6922,46 +6921,6 @@ return SDValue(); } -SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op, - SelectionDAG &DAG) const { - SDValue LHS = Op.getOperand(0); - EVT VT = Op.getValueType(); - - BuildVectorSDNode *BVN = - dyn_cast(Op.getOperand(1).getNode()); - if (!BVN) { - // AND commutes, so try swapping the operands. - LHS = Op.getOperand(1); - BVN = dyn_cast(Op.getOperand(0).getNode()); - } - if (!BVN) - return Op; - - APInt DefBits(VT.getSizeInBits(), 0); - APInt UndefBits(VT.getSizeInBits(), 0); - if (resolveBuildVector(BVN, DefBits, UndefBits)) { - SDValue NewOp; - - // We only have BIC vector immediate instruction, which is and-not. - DefBits = ~DefBits; - if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG, - DefBits, &LHS)) || - (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG, - DefBits, &LHS))) - return NewOp; - - UndefBits = ~UndefBits; - if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG, - UndefBits, &LHS)) || - (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG, - UndefBits, &LHS))) - return NewOp; - } - - // We can always fall back to a non-immediate AND. - return Op; -} - // Specialized code to quickly find if PotentialBVec is a BuildVector that // consists of only the same constant int value, returned in reference arg // ConstVal @@ -9433,6 +9392,46 @@ return SDValue(); } +static SDValue performANDCombine(SDNode *N, + TargetLowering::DAGCombinerInfo &DCI) { + SelectionDAG &DAG = DCI.DAG; + SDValue LHS = N->getOperand(0); + EVT VT = N->getValueType(0); + if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) + return SDValue(); + + BuildVectorSDNode *BVN = + dyn_cast(N->getOperand(1).getNode()); + if (!BVN) + return SDValue(); + + // AND does not accept an immediate, so check if we can use a BIC immediate + // instruction instead. We do this here instead of using a (and x, (mvni imm)) + // pattern in isel, because some immediates may be lowered to the preferred + // (and x, (movi imm)) form, even though an mvni representation also exists. + APInt DefBits(VT.getSizeInBits(), 0); + APInt UndefBits(VT.getSizeInBits(), 0); + if (resolveBuildVector(BVN, DefBits, UndefBits)) { + SDValue NewOp; + + DefBits = ~DefBits; + if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG, + DefBits, &LHS)) || + (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG, + DefBits, &LHS))) + return NewOp; + + UndefBits = ~UndefBits; + if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG, + UndefBits, &LHS)) || + (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG, + UndefBits, &LHS))) + return NewOp; + } + + return SDValue(); +} + static SDValue performSRLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { SelectionDAG &DAG = DCI.DAG; @@ -11283,6 +11282,8 @@ return performFDivCombine(N, DAG, DCI, Subtarget); case ISD::OR: return performORCombine(N, DCI, Subtarget); + case ISD::AND: + return performANDCombine(N, DCI); case ISD::SRL: return performSRLCombine(N, DCI); case ISD::INTRINSIC_WO_CHAIN: Index: llvm/test/CodeGen/AArch64/sadd_sat.ll =================================================================== --- llvm/test/CodeGen/AArch64/sadd_sat.ll +++ llvm/test/CodeGen/AArch64/sadd_sat.ll @@ -56,15 +56,14 @@ ; CHECK-NEXT: add v2.4s, v0.4s, v1.4s ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 ; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: cmge v4.4s, v2.4s, #0 -; CHECK-NEXT: movi v3.4s, #128, lsl #24 -; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s -; CHECK-NEXT: cmeq v0.4s, v0.4s, v4.4s +; CHECK-NEXT: cmge v5.4s, v2.4s, #0 ; CHECK-NEXT: cmlt v4.4s, v2.4s, #0 -; CHECK-NEXT: bic v3.16b, v3.16b, v4.16b -; CHECK-NEXT: bic v4.4s, #128, lsl #24 +; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s +; CHECK-NEXT: cmeq v0.4s, v0.4s, v5.4s +; CHECK-NEXT: mvni v3.4s, #128, lsl #24 +; CHECK-NEXT: mvn v5.16b, v4.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v3.16b, v4.16b, v5.16b ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b ; CHECK-NEXT: bsl v0.16b, v3.16b, v2.16b ; CHECK-NEXT: ret Index: llvm/test/CodeGen/AArch64/sadd_sat_vec.ll =================================================================== --- llvm/test/CodeGen/AArch64/sadd_sat_vec.ll +++ llvm/test/CodeGen/AArch64/sadd_sat_vec.ll @@ -153,15 +153,14 @@ ; CHECK-NEXT: add v2.8h, v0.8h, v1.8h ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 ; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: cmge v4.8h, v2.8h, #0 -; CHECK-NEXT: movi v3.8h, #128, lsl #8 -; CHECK-NEXT: cmeq v1.8h, v0.8h, v1.8h -; CHECK-NEXT: cmeq v0.8h, v0.8h, v4.8h +; CHECK-NEXT: cmge v5.8h, v2.8h, #0 ; CHECK-NEXT: cmlt v4.8h, v2.8h, #0 -; CHECK-NEXT: bic v3.16b, v3.16b, v4.16b -; CHECK-NEXT: bic v4.8h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.8h, v0.8h, v1.8h +; CHECK-NEXT: cmeq v0.8h, v0.8h, v5.8h +; CHECK-NEXT: mvni v3.8h, #128, lsl #8 +; CHECK-NEXT: mvn v5.16b, v4.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v3.16b, v4.16b, v5.16b ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b ; CHECK-NEXT: bsl v0.16b, v3.16b, v2.16b ; CHECK-NEXT: ret @@ -173,32 +172,31 @@ ; CHECK-LABEL: v16i16: ; CHECK: // %bb.0: ; CHECK-NEXT: add v4.8h, v0.8h, v2.8h +; CHECK-NEXT: cmlt v16.8h, v4.8h, #0 +; CHECK-NEXT: mvni v6.8h, #128, lsl #8 +; CHECK-NEXT: add v7.8h, v1.8h, v3.8h +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v6.16b, v16.16b, v17.16b +; CHECK-NEXT: cmlt v16.8h, v7.8h, #0 +; CHECK-NEXT: mvni v5.8h, #128, lsl #8 +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v17.16b ; CHECK-NEXT: cmge v2.8h, v2.8h, #0 ; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: add v6.8h, v1.8h, v3.8h ; CHECK-NEXT: cmge v16.8h, v4.8h, #0 -; CHECK-NEXT: movi v5.8h, #128, lsl #8 ; CHECK-NEXT: cmge v3.8h, v3.8h, #0 ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 -; CHECK-NEXT: cmlt v7.8h, v4.8h, #0 ; CHECK-NEXT: cmeq v2.8h, v0.8h, v2.8h ; CHECK-NEXT: cmeq v0.8h, v0.8h, v16.8h -; CHECK-NEXT: cmge v16.8h, v6.8h, #0 +; CHECK-NEXT: cmge v16.8h, v7.8h, #0 ; CHECK-NEXT: cmeq v3.8h, v1.8h, v3.8h ; CHECK-NEXT: cmeq v1.8h, v1.8h, v16.8h -; CHECK-NEXT: bic v16.16b, v5.16b, v7.16b -; CHECK-NEXT: bic v7.8h, #128, lsl #8 -; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b -; CHECK-NEXT: cmlt v16.8h, v6.8h, #0 -; CHECK-NEXT: bic v5.16b, v5.16b, v16.16b -; CHECK-NEXT: bic v16.8h, #128, lsl #8 ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: orr v5.16b, v16.16b, v5.16b ; CHECK-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-NEXT: and v1.16b, v3.16b, v1.16b -; CHECK-NEXT: bsl v0.16b, v7.16b, v4.16b -; CHECK-NEXT: bsl v1.16b, v5.16b, v6.16b +; CHECK-NEXT: bsl v0.16b, v6.16b, v4.16b +; CHECK-NEXT: bsl v1.16b, v5.16b, v7.16b ; CHECK-NEXT: ret %z = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %x, <16 x i16> %y) ret <16 x i16> %z @@ -207,59 +205,58 @@ define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind { ; CHECK-LABEL: v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: add v18.8h, v1.8h, v5.8h +; CHECK-NEXT: add v16.8h, v0.8h, v4.8h +; CHECK-NEXT: cmlt v24.8h, v16.8h, #0 +; CHECK-NEXT: mvni v18.8h, #128, lsl #8 +; CHECK-NEXT: add v19.8h, v1.8h, v5.8h +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v18.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.8h, v19.8h, #0 +; CHECK-NEXT: mvni v20.8h, #128, lsl #8 +; CHECK-NEXT: add v21.8h, v2.8h, v6.8h +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v20.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.8h, v21.8h, #0 +; CHECK-NEXT: mvni v22.8h, #128, lsl #8 +; CHECK-NEXT: add v23.8h, v3.8h, v7.8h +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v22.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.8h, v23.8h, #0 +; CHECK-NEXT: mvni v17.8h, #128, lsl #8 +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v17.16b, v24.16b, v25.16b +; CHECK-NEXT: cmge v4.8h, v4.8h, #0 +; CHECK-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-NEXT: cmge v24.8h, v16.8h, #0 ; CHECK-NEXT: cmge v5.8h, v5.8h, #0 ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 -; CHECK-NEXT: add v19.8h, v2.8h, v6.8h -; CHECK-NEXT: cmge v24.8h, v18.8h, #0 -; CHECK-NEXT: add v16.8h, v0.8h, v4.8h +; CHECK-NEXT: cmeq v4.8h, v0.8h, v4.8h +; CHECK-NEXT: cmeq v0.8h, v0.8h, v24.8h +; CHECK-NEXT: cmge v24.8h, v19.8h, #0 ; CHECK-NEXT: cmge v6.8h, v6.8h, #0 ; CHECK-NEXT: cmge v2.8h, v2.8h, #0 -; CHECK-NEXT: add v20.8h, v3.8h, v7.8h ; CHECK-NEXT: cmeq v5.8h, v1.8h, v5.8h ; CHECK-NEXT: cmeq v1.8h, v1.8h, v24.8h -; CHECK-NEXT: cmge v24.8h, v19.8h, #0 -; CHECK-NEXT: movi v17.8h, #128, lsl #8 +; CHECK-NEXT: cmge v24.8h, v21.8h, #0 ; CHECK-NEXT: cmge v7.8h, v7.8h, #0 ; CHECK-NEXT: cmge v3.8h, v3.8h, #0 -; CHECK-NEXT: cmlt v21.8h, v16.8h, #0 ; CHECK-NEXT: cmeq v6.8h, v2.8h, v6.8h ; CHECK-NEXT: cmeq v2.8h, v2.8h, v24.8h -; CHECK-NEXT: cmge v24.8h, v20.8h, #0 -; CHECK-NEXT: cmge v4.8h, v4.8h, #0 -; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: cmge v22.8h, v16.8h, #0 -; CHECK-NEXT: cmlt v23.8h, v18.8h, #0 +; CHECK-NEXT: cmge v24.8h, v23.8h, #0 ; CHECK-NEXT: cmeq v7.8h, v3.8h, v7.8h ; CHECK-NEXT: cmeq v3.8h, v3.8h, v24.8h -; CHECK-NEXT: bic v24.16b, v17.16b, v21.16b -; CHECK-NEXT: bic v21.8h, #128, lsl #8 -; CHECK-NEXT: cmeq v4.8h, v0.8h, v4.8h -; CHECK-NEXT: cmeq v0.8h, v0.8h, v22.8h -; CHECK-NEXT: cmlt v22.8h, v19.8h, #0 -; CHECK-NEXT: orr v21.16b, v21.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v23.16b -; CHECK-NEXT: bic v23.8h, #128, lsl #8 -; CHECK-NEXT: orr v23.16b, v23.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v22.16b -; CHECK-NEXT: bic v22.8h, #128, lsl #8 -; CHECK-NEXT: orr v22.16b, v22.16b, v24.16b -; CHECK-NEXT: cmlt v24.8h, v20.8h, #0 -; CHECK-NEXT: bic v17.16b, v17.16b, v24.16b -; CHECK-NEXT: bic v24.8h, #128, lsl #8 ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v2.16b, v2.16b ; CHECK-NEXT: mvn v3.16b, v3.16b -; CHECK-NEXT: orr v17.16b, v24.16b, v17.16b ; CHECK-NEXT: and v0.16b, v4.16b, v0.16b ; CHECK-NEXT: and v1.16b, v5.16b, v1.16b ; CHECK-NEXT: and v2.16b, v6.16b, v2.16b ; CHECK-NEXT: and v3.16b, v7.16b, v3.16b -; CHECK-NEXT: bsl v0.16b, v21.16b, v16.16b -; CHECK-NEXT: bsl v1.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v2.16b, v22.16b, v19.16b -; CHECK-NEXT: bsl v3.16b, v17.16b, v20.16b +; CHECK-NEXT: bsl v0.16b, v18.16b, v16.16b +; CHECK-NEXT: bsl v1.16b, v20.16b, v19.16b +; CHECK-NEXT: bsl v2.16b, v22.16b, v21.16b +; CHECK-NEXT: bsl v3.16b, v17.16b, v23.16b ; CHECK-NEXT: ret %z = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %x, <32 x i16> %y) ret <32 x i16> %z @@ -316,15 +313,14 @@ ; CHECK-NEXT: add v3.4h, v0.4h, v1.4h ; CHECK-NEXT: cmge v1.4h, v1.4h, #0 ; CHECK-NEXT: cmge v0.4h, v0.4h, #0 -; CHECK-NEXT: cmge v4.4h, v3.4h, #0 -; CHECK-NEXT: movi v2.4h, #128, lsl #8 -; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h -; CHECK-NEXT: cmeq v0.4h, v0.4h, v4.4h +; CHECK-NEXT: cmge v5.4h, v3.4h, #0 ; CHECK-NEXT: cmlt v4.4h, v3.4h, #0 -; CHECK-NEXT: bic v2.8b, v2.8b, v4.8b -; CHECK-NEXT: bic v4.4h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h +; CHECK-NEXT: cmeq v0.4h, v0.4h, v5.4h +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v2.8b, v4.8b, v2.8b +; CHECK-NEXT: bsl v2.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 @@ -354,15 +350,14 @@ ; CHECK-NEXT: add v3.2s, v0.2s, v2.2s ; CHECK-NEXT: cmge v2.2s, v2.2s, #0 ; CHECK-NEXT: cmge v0.2s, v0.2s, #0 -; CHECK-NEXT: cmge v4.2s, v3.2s, #0 -; CHECK-NEXT: movi v1.2s, #128, lsl #24 -; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s -; CHECK-NEXT: cmeq v0.2s, v0.2s, v4.2s +; CHECK-NEXT: cmge v5.2s, v3.2s, #0 ; CHECK-NEXT: cmlt v4.2s, v3.2s, #0 -; CHECK-NEXT: bic v1.8b, v1.8b, v4.8b -; CHECK-NEXT: bic v4.2s, #128, lsl #24 +; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s +; CHECK-NEXT: cmeq v0.2s, v0.2s, v5.2s +; CHECK-NEXT: mvni v1.2s, #128, lsl #24 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v1.8b, v4.8b, v1.8b +; CHECK-NEXT: bsl v1.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v2.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v1.8b, v3.8b ; CHECK-NEXT: ushr v0.2s, v0.2s, #24 @@ -383,18 +378,17 @@ ; CHECK: // %bb.0: ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] -; CHECK-NEXT: movi v2.4h, #128, lsl #8 +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 ; CHECK-NEXT: add v3.4h, v0.4h, v1.4h ; CHECK-NEXT: cmge v1.4h, v1.4h, #0 ; CHECK-NEXT: cmge v0.4h, v0.4h, #0 -; CHECK-NEXT: cmge v4.4h, v3.4h, #0 -; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h -; CHECK-NEXT: cmeq v0.4h, v0.4h, v4.4h +; CHECK-NEXT: cmge v5.4h, v3.4h, #0 ; CHECK-NEXT: cmlt v4.4h, v3.4h, #0 -; CHECK-NEXT: bic v2.8b, v2.8b, v4.8b -; CHECK-NEXT: bic v4.4h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h +; CHECK-NEXT: cmeq v0.4h, v0.4h, v5.4h +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v2.8b, v4.8b, v2.8b +; CHECK-NEXT: bsl v2.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b ; CHECK-NEXT: str d0, [x2] @@ -422,15 +416,14 @@ ; CHECK-NEXT: add v3.2s, v0.2s, v2.2s ; CHECK-NEXT: cmge v2.2s, v2.2s, #0 ; CHECK-NEXT: cmge v0.2s, v0.2s, #0 -; CHECK-NEXT: cmge v4.2s, v3.2s, #0 -; CHECK-NEXT: movi v1.2s, #128, lsl #24 -; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s -; CHECK-NEXT: cmeq v0.2s, v0.2s, v4.2s +; CHECK-NEXT: cmge v5.2s, v3.2s, #0 ; CHECK-NEXT: cmlt v4.2s, v3.2s, #0 -; CHECK-NEXT: bic v1.8b, v1.8b, v4.8b -; CHECK-NEXT: bic v4.2s, #128, lsl #24 +; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s +; CHECK-NEXT: cmeq v0.2s, v0.2s, v5.2s +; CHECK-NEXT: mvni v1.2s, #128, lsl #24 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v1.8b, v4.8b, v1.8b +; CHECK-NEXT: bsl v1.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v2.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v1.8b, v3.8b ; CHECK-NEXT: ushr v0.2s, v0.2s, #16 @@ -470,37 +463,36 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind { ; CHECK-LABEL: v12i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q2, q0, [x0] -; CHECK-NEXT: ldp q3, q1, [x1] -; CHECK-NEXT: movi v4.8h, #128, lsl #8 -; CHECK-NEXT: add v5.8h, v0.8h, v1.8h +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q3, q2, [x1] +; CHECK-NEXT: mvni v5.8h, #128, lsl #8 +; CHECK-NEXT: mvni v4.8h, #128, lsl #8 +; CHECK-NEXT: add v6.8h, v1.8h, v2.8h +; CHECK-NEXT: cmlt v16.8h, v6.8h, #0 +; CHECK-NEXT: add v7.8h, v0.8h, v3.8h +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v17.16b +; CHECK-NEXT: cmlt v16.8h, v7.8h, #0 +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v4.16b, v16.16b, v17.16b +; CHECK-NEXT: cmge v2.8h, v2.8h, #0 ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 -; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: add v6.8h, v2.8h, v3.8h -; CHECK-NEXT: cmge v16.8h, v5.8h, #0 +; CHECK-NEXT: cmge v16.8h, v6.8h, #0 ; CHECK-NEXT: cmge v3.8h, v3.8h, #0 -; CHECK-NEXT: cmge v2.8h, v2.8h, #0 -; CHECK-NEXT: cmlt v7.8h, v5.8h, #0 -; CHECK-NEXT: cmeq v1.8h, v0.8h, v1.8h +; CHECK-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-NEXT: cmeq v2.8h, v1.8h, v2.8h +; CHECK-NEXT: cmeq v1.8h, v1.8h, v16.8h +; CHECK-NEXT: cmge v16.8h, v7.8h, #0 +; CHECK-NEXT: cmeq v3.8h, v0.8h, v3.8h ; CHECK-NEXT: cmeq v0.8h, v0.8h, v16.8h -; CHECK-NEXT: cmge v16.8h, v6.8h, #0 -; CHECK-NEXT: cmeq v3.8h, v2.8h, v3.8h -; CHECK-NEXT: cmeq v2.8h, v2.8h, v16.8h -; CHECK-NEXT: bic v16.16b, v4.16b, v7.16b -; CHECK-NEXT: bic v7.8h, #128, lsl #8 -; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b -; CHECK-NEXT: cmlt v16.8h, v6.8h, #0 -; CHECK-NEXT: bic v4.16b, v4.16b, v16.16b -; CHECK-NEXT: bic v16.8h, #128, lsl #8 +; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: mvn v2.16b, v2.16b -; CHECK-NEXT: orr v4.16b, v16.16b, v4.16b -; CHECK-NEXT: and v0.16b, v1.16b, v0.16b -; CHECK-NEXT: and v1.16b, v3.16b, v2.16b -; CHECK-NEXT: bsl v0.16b, v7.16b, v5.16b -; CHECK-NEXT: bsl v1.16b, v4.16b, v6.16b -; CHECK-NEXT: str q1, [x2] -; CHECK-NEXT: str d0, [x2, #16] +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: and v0.16b, v3.16b, v0.16b +; CHECK-NEXT: bsl v1.16b, v5.16b, v6.16b +; CHECK-NEXT: bsl v0.16b, v4.16b, v7.16b +; CHECK-NEXT: str q0, [x2] +; CHECK-NEXT: str d1, [x2, #16] ; CHECK-NEXT: ret %x = load <12 x i16>, <12 x i16>* %px %y = load <12 x i16>, <12 x i16>* %py @@ -541,18 +533,17 @@ ; CHECK: // %bb.0: ; CHECK-NEXT: ldr h0, [x0] ; CHECK-NEXT: ldr h1, [x1] -; CHECK-NEXT: movi v2.4h, #128, lsl #8 +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 ; CHECK-NEXT: add v3.4h, v0.4h, v1.4h ; CHECK-NEXT: cmge v1.4h, v1.4h, #0 ; CHECK-NEXT: cmge v0.4h, v0.4h, #0 -; CHECK-NEXT: cmge v4.4h, v3.4h, #0 -; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h -; CHECK-NEXT: cmeq v0.4h, v0.4h, v4.4h +; CHECK-NEXT: cmge v5.4h, v3.4h, #0 ; CHECK-NEXT: cmlt v4.4h, v3.4h, #0 -; CHECK-NEXT: bic v2.8b, v2.8b, v4.8b -; CHECK-NEXT: bic v4.4h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h +; CHECK-NEXT: cmeq v0.4h, v0.4h, v5.4h +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v2.8b, v4.8b, v2.8b +; CHECK-NEXT: bsl v2.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b ; CHECK-NEXT: str h0, [x2] @@ -618,15 +609,14 @@ ; CHECK-NEXT: add v2.2s, v0.2s, v1.2s ; CHECK-NEXT: cmge v1.2s, v1.2s, #0 ; CHECK-NEXT: cmge v0.2s, v0.2s, #0 -; CHECK-NEXT: cmge v4.2s, v2.2s, #0 -; CHECK-NEXT: movi v3.2s, #128, lsl #24 -; CHECK-NEXT: cmeq v1.2s, v0.2s, v1.2s -; CHECK-NEXT: cmeq v0.2s, v0.2s, v4.2s +; CHECK-NEXT: cmge v5.2s, v2.2s, #0 ; CHECK-NEXT: cmlt v4.2s, v2.2s, #0 -; CHECK-NEXT: bic v3.8b, v3.8b, v4.8b -; CHECK-NEXT: bic v4.2s, #128, lsl #24 +; CHECK-NEXT: cmeq v1.2s, v0.2s, v1.2s +; CHECK-NEXT: cmeq v0.2s, v0.2s, v5.2s +; CHECK-NEXT: mvni v3.2s, #128, lsl #24 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v3.8b, v4.8b, v3.8b +; CHECK-NEXT: bsl v3.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v3.8b, v2.8b ; CHECK-NEXT: ret @@ -640,15 +630,14 @@ ; CHECK-NEXT: add v2.4s, v0.4s, v1.4s ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 ; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: cmge v4.4s, v2.4s, #0 -; CHECK-NEXT: movi v3.4s, #128, lsl #24 -; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s -; CHECK-NEXT: cmeq v0.4s, v0.4s, v4.4s +; CHECK-NEXT: cmge v5.4s, v2.4s, #0 ; CHECK-NEXT: cmlt v4.4s, v2.4s, #0 -; CHECK-NEXT: bic v3.16b, v3.16b, v4.16b -; CHECK-NEXT: bic v4.4s, #128, lsl #24 +; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s +; CHECK-NEXT: cmeq v0.4s, v0.4s, v5.4s +; CHECK-NEXT: mvni v3.4s, #128, lsl #24 +; CHECK-NEXT: mvn v5.16b, v4.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v3.16b, v4.16b, v5.16b ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b ; CHECK-NEXT: bsl v0.16b, v3.16b, v2.16b ; CHECK-NEXT: ret @@ -660,32 +649,31 @@ ; CHECK-LABEL: v8i32: ; CHECK: // %bb.0: ; CHECK-NEXT: add v4.4s, v0.4s, v2.4s +; CHECK-NEXT: cmlt v16.4s, v4.4s, #0 +; CHECK-NEXT: mvni v6.4s, #128, lsl #24 +; CHECK-NEXT: add v7.4s, v1.4s, v3.4s +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v6.16b, v16.16b, v17.16b +; CHECK-NEXT: cmlt v16.4s, v7.4s, #0 +; CHECK-NEXT: mvni v5.4s, #128, lsl #24 +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v17.16b ; CHECK-NEXT: cmge v2.4s, v2.4s, #0 ; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: add v6.4s, v1.4s, v3.4s ; CHECK-NEXT: cmge v16.4s, v4.4s, #0 -; CHECK-NEXT: movi v5.4s, #128, lsl #24 ; CHECK-NEXT: cmge v3.4s, v3.4s, #0 ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 -; CHECK-NEXT: cmlt v7.4s, v4.4s, #0 ; CHECK-NEXT: cmeq v2.4s, v0.4s, v2.4s ; CHECK-NEXT: cmeq v0.4s, v0.4s, v16.4s -; CHECK-NEXT: cmge v16.4s, v6.4s, #0 +; CHECK-NEXT: cmge v16.4s, v7.4s, #0 ; CHECK-NEXT: cmeq v3.4s, v1.4s, v3.4s ; CHECK-NEXT: cmeq v1.4s, v1.4s, v16.4s -; CHECK-NEXT: bic v16.16b, v5.16b, v7.16b -; CHECK-NEXT: bic v7.4s, #128, lsl #24 -; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b -; CHECK-NEXT: cmlt v16.4s, v6.4s, #0 -; CHECK-NEXT: bic v5.16b, v5.16b, v16.16b -; CHECK-NEXT: bic v16.4s, #128, lsl #24 ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: orr v5.16b, v16.16b, v5.16b ; CHECK-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-NEXT: and v1.16b, v3.16b, v1.16b -; CHECK-NEXT: bsl v0.16b, v7.16b, v4.16b -; CHECK-NEXT: bsl v1.16b, v5.16b, v6.16b +; CHECK-NEXT: bsl v0.16b, v6.16b, v4.16b +; CHECK-NEXT: bsl v1.16b, v5.16b, v7.16b ; CHECK-NEXT: ret %z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y) ret <8 x i32> %z @@ -694,59 +682,58 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind { ; CHECK-LABEL: v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: add v18.4s, v1.4s, v5.4s +; CHECK-NEXT: add v16.4s, v0.4s, v4.4s +; CHECK-NEXT: cmlt v24.4s, v16.4s, #0 +; CHECK-NEXT: mvni v18.4s, #128, lsl #24 +; CHECK-NEXT: add v19.4s, v1.4s, v5.4s +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v18.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.4s, v19.4s, #0 +; CHECK-NEXT: mvni v20.4s, #128, lsl #24 +; CHECK-NEXT: add v21.4s, v2.4s, v6.4s +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v20.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.4s, v21.4s, #0 +; CHECK-NEXT: mvni v22.4s, #128, lsl #24 +; CHECK-NEXT: add v23.4s, v3.4s, v7.4s +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v22.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.4s, v23.4s, #0 +; CHECK-NEXT: mvni v17.4s, #128, lsl #24 +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v17.16b, v24.16b, v25.16b +; CHECK-NEXT: cmge v4.4s, v4.4s, #0 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: cmge v24.4s, v16.4s, #0 ; CHECK-NEXT: cmge v5.4s, v5.4s, #0 ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 -; CHECK-NEXT: add v19.4s, v2.4s, v6.4s -; CHECK-NEXT: cmge v24.4s, v18.4s, #0 -; CHECK-NEXT: add v16.4s, v0.4s, v4.4s +; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s +; CHECK-NEXT: cmeq v0.4s, v0.4s, v24.4s +; CHECK-NEXT: cmge v24.4s, v19.4s, #0 ; CHECK-NEXT: cmge v6.4s, v6.4s, #0 ; CHECK-NEXT: cmge v2.4s, v2.4s, #0 -; CHECK-NEXT: add v20.4s, v3.4s, v7.4s ; CHECK-NEXT: cmeq v5.4s, v1.4s, v5.4s ; CHECK-NEXT: cmeq v1.4s, v1.4s, v24.4s -; CHECK-NEXT: cmge v24.4s, v19.4s, #0 -; CHECK-NEXT: movi v17.4s, #128, lsl #24 +; CHECK-NEXT: cmge v24.4s, v21.4s, #0 ; CHECK-NEXT: cmge v7.4s, v7.4s, #0 ; CHECK-NEXT: cmge v3.4s, v3.4s, #0 -; CHECK-NEXT: cmlt v21.4s, v16.4s, #0 ; CHECK-NEXT: cmeq v6.4s, v2.4s, v6.4s ; CHECK-NEXT: cmeq v2.4s, v2.4s, v24.4s -; CHECK-NEXT: cmge v24.4s, v20.4s, #0 -; CHECK-NEXT: cmge v4.4s, v4.4s, #0 -; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: cmge v22.4s, v16.4s, #0 -; CHECK-NEXT: cmlt v23.4s, v18.4s, #0 +; CHECK-NEXT: cmge v24.4s, v23.4s, #0 ; CHECK-NEXT: cmeq v7.4s, v3.4s, v7.4s ; CHECK-NEXT: cmeq v3.4s, v3.4s, v24.4s -; CHECK-NEXT: bic v24.16b, v17.16b, v21.16b -; CHECK-NEXT: bic v21.4s, #128, lsl #24 -; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s -; CHECK-NEXT: cmeq v0.4s, v0.4s, v22.4s -; CHECK-NEXT: cmlt v22.4s, v19.4s, #0 -; CHECK-NEXT: orr v21.16b, v21.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v23.16b -; CHECK-NEXT: bic v23.4s, #128, lsl #24 -; CHECK-NEXT: orr v23.16b, v23.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v22.16b -; CHECK-NEXT: bic v22.4s, #128, lsl #24 -; CHECK-NEXT: orr v22.16b, v22.16b, v24.16b -; CHECK-NEXT: cmlt v24.4s, v20.4s, #0 -; CHECK-NEXT: bic v17.16b, v17.16b, v24.16b -; CHECK-NEXT: bic v24.4s, #128, lsl #24 ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v2.16b, v2.16b ; CHECK-NEXT: mvn v3.16b, v3.16b -; CHECK-NEXT: orr v17.16b, v24.16b, v17.16b ; CHECK-NEXT: and v0.16b, v4.16b, v0.16b ; CHECK-NEXT: and v1.16b, v5.16b, v1.16b ; CHECK-NEXT: and v2.16b, v6.16b, v2.16b ; CHECK-NEXT: and v3.16b, v7.16b, v3.16b -; CHECK-NEXT: bsl v0.16b, v21.16b, v16.16b -; CHECK-NEXT: bsl v1.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v2.16b, v22.16b, v19.16b -; CHECK-NEXT: bsl v3.16b, v17.16b, v20.16b +; CHECK-NEXT: bsl v0.16b, v18.16b, v16.16b +; CHECK-NEXT: bsl v1.16b, v20.16b, v19.16b +; CHECK-NEXT: bsl v2.16b, v22.16b, v21.16b +; CHECK-NEXT: bsl v3.16b, v17.16b, v23.16b ; CHECK-NEXT: ret %z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y) ret <16 x i32> %z Index: llvm/test/CodeGen/AArch64/sat-add.ll =================================================================== --- llvm/test/CodeGen/AArch64/sat-add.ll +++ llvm/test/CodeGen/AArch64/sat-add.ll @@ -365,7 +365,6 @@ ; CHECK-NEXT: add v1.16b, v0.16b, v1.16b ; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <16 x i8> %x, @@ -382,7 +381,6 @@ ; CHECK-NEXT: add v1.16b, v0.16b, v1.16b ; CHECK-NEXT: cmhi v0.16b, v0.16b, v2.16b ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <16 x i8> %x, @@ -412,7 +410,6 @@ ; CHECK-NEXT: add v1.8h, v0.8h, v1.8h ; CHECK-NEXT: cmhi v0.8h, v0.8h, v1.8h ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <8 x i16> %x, @@ -429,7 +426,6 @@ ; CHECK-NEXT: add v1.8h, v0.8h, v1.8h ; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <8 x i16> %x, @@ -459,7 +455,6 @@ ; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <4 x i32> %x, @@ -476,7 +471,6 @@ ; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmhi v0.4s, v0.4s, v2.4s ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <4 x i32> %x, @@ -510,7 +504,6 @@ ; CHECK-NEXT: add v1.2d, v0.2d, v1.2d ; CHECK-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <2 x i64> %x, @@ -529,7 +522,6 @@ ; CHECK-NEXT: add v1.2d, v0.2d, v1.2d ; CHECK-NEXT: cmhi v0.2d, v0.2d, v2.2d ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <2 x i64> %x, @@ -558,7 +550,6 @@ ; CHECK-NEXT: add v1.16b, v0.16b, v1.16b ; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <16 x i8> %x, %y @@ -574,7 +565,6 @@ ; CHECK-NEXT: add v1.16b, v0.16b, v1.16b ; CHECK-NEXT: cmhi v0.16b, v0.16b, v2.16b ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %noty = xor <16 x i8> %y, @@ -604,7 +594,6 @@ ; CHECK-NEXT: add v1.8h, v0.8h, v1.8h ; CHECK-NEXT: cmhi v0.8h, v0.8h, v1.8h ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <8 x i16> %x, %y @@ -620,7 +609,6 @@ ; CHECK-NEXT: add v1.8h, v0.8h, v1.8h ; CHECK-NEXT: cmhi v0.8h, v0.8h, v2.8h ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %noty = xor <8 x i16> %y, @@ -650,7 +638,6 @@ ; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <4 x i32> %x, %y @@ -666,7 +653,6 @@ ; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmhi v0.4s, v0.4s, v2.4s ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %noty = xor <4 x i32> %y, @@ -697,7 +683,6 @@ ; CHECK-NEXT: add v1.2d, v0.2d, v1.2d ; CHECK-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %a = add <2 x i64> %x, %y @@ -713,7 +698,6 @@ ; CHECK-NEXT: add v1.2d, v0.2d, v1.2d ; CHECK-NEXT: cmhi v0.2d, v0.2d, v2.2d ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %noty = xor <2 x i64> %y, Index: llvm/test/CodeGen/AArch64/ssub_sat.ll =================================================================== --- llvm/test/CodeGen/AArch64/ssub_sat.ll +++ llvm/test/CodeGen/AArch64/ssub_sat.ll @@ -56,16 +56,15 @@ ; CHECK-NEXT: sub v2.4s, v0.4s, v1.4s ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 ; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: cmge v4.4s, v2.4s, #0 -; CHECK-NEXT: movi v3.4s, #128, lsl #24 -; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s -; CHECK-NEXT: cmeq v0.4s, v0.4s, v4.4s +; CHECK-NEXT: cmge v5.4s, v2.4s, #0 ; CHECK-NEXT: cmlt v4.4s, v2.4s, #0 -; CHECK-NEXT: bic v3.16b, v3.16b, v4.16b -; CHECK-NEXT: bic v4.4s, #128, lsl #24 +; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s +; CHECK-NEXT: cmeq v0.4s, v0.4s, v5.4s +; CHECK-NEXT: mvni v3.4s, #128, lsl #24 +; CHECK-NEXT: mvn v5.16b, v4.16b ; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v3.16b, v4.16b, v5.16b ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b ; CHECK-NEXT: bsl v0.16b, v3.16b, v2.16b ; CHECK-NEXT: ret Index: llvm/test/CodeGen/AArch64/ssub_sat_vec.ll =================================================================== --- llvm/test/CodeGen/AArch64/ssub_sat_vec.ll +++ llvm/test/CodeGen/AArch64/ssub_sat_vec.ll @@ -161,16 +161,15 @@ ; CHECK-NEXT: sub v2.8h, v0.8h, v1.8h ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 ; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: cmge v4.8h, v2.8h, #0 -; CHECK-NEXT: movi v3.8h, #128, lsl #8 -; CHECK-NEXT: cmeq v1.8h, v0.8h, v1.8h -; CHECK-NEXT: cmeq v0.8h, v0.8h, v4.8h +; CHECK-NEXT: cmge v5.8h, v2.8h, #0 ; CHECK-NEXT: cmlt v4.8h, v2.8h, #0 -; CHECK-NEXT: bic v3.16b, v3.16b, v4.16b -; CHECK-NEXT: bic v4.8h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.8h, v0.8h, v1.8h +; CHECK-NEXT: cmeq v0.8h, v0.8h, v5.8h +; CHECK-NEXT: mvni v3.8h, #128, lsl #8 +; CHECK-NEXT: mvn v5.16b, v4.16b ; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v3.16b, v4.16b, v5.16b ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b ; CHECK-NEXT: bsl v0.16b, v3.16b, v2.16b ; CHECK-NEXT: ret @@ -182,34 +181,33 @@ ; CHECK-LABEL: v16i16: ; CHECK: // %bb.0: ; CHECK-NEXT: sub v4.8h, v0.8h, v2.8h +; CHECK-NEXT: cmlt v16.8h, v4.8h, #0 +; CHECK-NEXT: mvni v6.8h, #128, lsl #8 +; CHECK-NEXT: sub v7.8h, v1.8h, v3.8h +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v6.16b, v16.16b, v17.16b +; CHECK-NEXT: cmlt v16.8h, v7.8h, #0 +; CHECK-NEXT: mvni v5.8h, #128, lsl #8 +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v17.16b ; CHECK-NEXT: cmge v2.8h, v2.8h, #0 ; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: sub v6.8h, v1.8h, v3.8h ; CHECK-NEXT: cmge v16.8h, v4.8h, #0 -; CHECK-NEXT: movi v5.8h, #128, lsl #8 ; CHECK-NEXT: cmge v3.8h, v3.8h, #0 ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 -; CHECK-NEXT: cmlt v7.8h, v4.8h, #0 ; CHECK-NEXT: cmeq v2.8h, v0.8h, v2.8h ; CHECK-NEXT: cmeq v0.8h, v0.8h, v16.8h -; CHECK-NEXT: cmge v16.8h, v6.8h, #0 +; CHECK-NEXT: cmge v16.8h, v7.8h, #0 ; CHECK-NEXT: cmeq v3.8h, v1.8h, v3.8h ; CHECK-NEXT: cmeq v1.8h, v1.8h, v16.8h -; CHECK-NEXT: bic v16.16b, v5.16b, v7.16b -; CHECK-NEXT: bic v7.8h, #128, lsl #8 -; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b -; CHECK-NEXT: cmlt v16.8h, v6.8h, #0 -; CHECK-NEXT: bic v5.16b, v5.16b, v16.16b -; CHECK-NEXT: bic v16.8h, #128, lsl #8 ; CHECK-NEXT: mvn v2.16b, v2.16b ; CHECK-NEXT: mvn v3.16b, v3.16b ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: orr v5.16b, v16.16b, v5.16b ; CHECK-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-NEXT: and v1.16b, v3.16b, v1.16b -; CHECK-NEXT: bsl v0.16b, v7.16b, v4.16b -; CHECK-NEXT: bsl v1.16b, v5.16b, v6.16b +; CHECK-NEXT: bsl v0.16b, v6.16b, v4.16b +; CHECK-NEXT: bsl v1.16b, v5.16b, v7.16b ; CHECK-NEXT: ret %z = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %x, <16 x i16> %y) ret <16 x i16> %z @@ -218,63 +216,62 @@ define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind { ; CHECK-LABEL: v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v18.8h, v1.8h, v5.8h +; CHECK-NEXT: sub v16.8h, v0.8h, v4.8h +; CHECK-NEXT: cmlt v24.8h, v16.8h, #0 +; CHECK-NEXT: mvni v18.8h, #128, lsl #8 +; CHECK-NEXT: sub v19.8h, v1.8h, v5.8h +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v18.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.8h, v19.8h, #0 +; CHECK-NEXT: mvni v20.8h, #128, lsl #8 +; CHECK-NEXT: sub v21.8h, v2.8h, v6.8h +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v20.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.8h, v21.8h, #0 +; CHECK-NEXT: mvni v22.8h, #128, lsl #8 +; CHECK-NEXT: sub v23.8h, v3.8h, v7.8h +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v22.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.8h, v23.8h, #0 +; CHECK-NEXT: mvni v17.8h, #128, lsl #8 +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v17.16b, v24.16b, v25.16b +; CHECK-NEXT: cmge v4.8h, v4.8h, #0 +; CHECK-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-NEXT: cmge v24.8h, v16.8h, #0 ; CHECK-NEXT: cmge v5.8h, v5.8h, #0 ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 -; CHECK-NEXT: sub v19.8h, v2.8h, v6.8h -; CHECK-NEXT: cmge v24.8h, v18.8h, #0 -; CHECK-NEXT: sub v16.8h, v0.8h, v4.8h +; CHECK-NEXT: cmeq v4.8h, v0.8h, v4.8h +; CHECK-NEXT: cmeq v0.8h, v0.8h, v24.8h +; CHECK-NEXT: cmge v24.8h, v19.8h, #0 ; CHECK-NEXT: cmge v6.8h, v6.8h, #0 ; CHECK-NEXT: cmge v2.8h, v2.8h, #0 -; CHECK-NEXT: sub v20.8h, v3.8h, v7.8h ; CHECK-NEXT: cmeq v5.8h, v1.8h, v5.8h ; CHECK-NEXT: cmeq v1.8h, v1.8h, v24.8h -; CHECK-NEXT: cmge v24.8h, v19.8h, #0 -; CHECK-NEXT: movi v17.8h, #128, lsl #8 -; CHECK-NEXT: cmge v4.8h, v4.8h, #0 -; CHECK-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-NEXT: cmge v24.8h, v21.8h, #0 +; CHECK-NEXT: mvn v4.16b, v4.16b +; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: cmge v7.8h, v7.8h, #0 ; CHECK-NEXT: cmge v3.8h, v3.8h, #0 -; CHECK-NEXT: cmlt v21.8h, v16.8h, #0 -; CHECK-NEXT: cmge v22.8h, v16.8h, #0 ; CHECK-NEXT: cmeq v6.8h, v2.8h, v6.8h ; CHECK-NEXT: cmeq v2.8h, v2.8h, v24.8h -; CHECK-NEXT: cmge v24.8h, v20.8h, #0 -; CHECK-NEXT: cmeq v4.8h, v0.8h, v4.8h -; CHECK-NEXT: cmlt v23.8h, v18.8h, #0 -; CHECK-NEXT: cmeq v0.8h, v0.8h, v22.8h -; CHECK-NEXT: cmeq v7.8h, v3.8h, v7.8h -; CHECK-NEXT: cmeq v3.8h, v3.8h, v24.8h -; CHECK-NEXT: bic v24.16b, v17.16b, v21.16b -; CHECK-NEXT: bic v21.8h, #128, lsl #8 -; CHECK-NEXT: cmlt v22.8h, v19.8h, #0 -; CHECK-NEXT: orr v21.16b, v21.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v23.16b -; CHECK-NEXT: bic v23.8h, #128, lsl #8 -; CHECK-NEXT: mvn v4.16b, v4.16b -; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v23.16b, v23.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v22.16b -; CHECK-NEXT: bic v22.8h, #128, lsl #8 +; CHECK-NEXT: cmge v24.8h, v23.8h, #0 ; CHECK-NEXT: and v0.16b, v4.16b, v0.16b ; CHECK-NEXT: mvn v4.16b, v5.16b ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: orr v22.16b, v22.16b, v24.16b -; CHECK-NEXT: cmlt v24.8h, v20.8h, #0 +; CHECK-NEXT: cmeq v7.8h, v3.8h, v7.8h +; CHECK-NEXT: cmeq v3.8h, v3.8h, v24.8h ; CHECK-NEXT: and v1.16b, v4.16b, v1.16b ; CHECK-NEXT: mvn v4.16b, v6.16b ; CHECK-NEXT: mvn v2.16b, v2.16b -; CHECK-NEXT: bic v17.16b, v17.16b, v24.16b -; CHECK-NEXT: bic v24.8h, #128, lsl #8 ; CHECK-NEXT: and v2.16b, v4.16b, v2.16b ; CHECK-NEXT: mvn v4.16b, v7.16b ; CHECK-NEXT: mvn v3.16b, v3.16b -; CHECK-NEXT: orr v17.16b, v24.16b, v17.16b ; CHECK-NEXT: and v3.16b, v4.16b, v3.16b -; CHECK-NEXT: bsl v0.16b, v21.16b, v16.16b -; CHECK-NEXT: bsl v1.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v2.16b, v22.16b, v19.16b -; CHECK-NEXT: bsl v3.16b, v17.16b, v20.16b +; CHECK-NEXT: bsl v0.16b, v18.16b, v16.16b +; CHECK-NEXT: bsl v1.16b, v20.16b, v19.16b +; CHECK-NEXT: bsl v2.16b, v22.16b, v21.16b +; CHECK-NEXT: bsl v3.16b, v17.16b, v23.16b ; CHECK-NEXT: ret %z = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %x, <32 x i16> %y) ret <32 x i16> %z @@ -332,16 +329,15 @@ ; CHECK-NEXT: sub v3.4h, v0.4h, v1.4h ; CHECK-NEXT: cmge v1.4h, v1.4h, #0 ; CHECK-NEXT: cmge v0.4h, v0.4h, #0 -; CHECK-NEXT: cmge v4.4h, v3.4h, #0 -; CHECK-NEXT: movi v2.4h, #128, lsl #8 -; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h -; CHECK-NEXT: cmeq v0.4h, v0.4h, v4.4h +; CHECK-NEXT: cmge v5.4h, v3.4h, #0 ; CHECK-NEXT: cmlt v4.4h, v3.4h, #0 -; CHECK-NEXT: bic v2.8b, v2.8b, v4.8b -; CHECK-NEXT: bic v4.4h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h +; CHECK-NEXT: cmeq v0.4h, v0.4h, v5.4h +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v1.8b, v1.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v2.8b, v4.8b, v2.8b +; CHECK-NEXT: bsl v2.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b ; CHECK-NEXT: sshr v0.4h, v0.4h, #8 @@ -371,16 +367,15 @@ ; CHECK-NEXT: sub v3.2s, v0.2s, v2.2s ; CHECK-NEXT: cmge v2.2s, v2.2s, #0 ; CHECK-NEXT: cmge v0.2s, v0.2s, #0 -; CHECK-NEXT: cmge v4.2s, v3.2s, #0 -; CHECK-NEXT: movi v1.2s, #128, lsl #24 -; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s -; CHECK-NEXT: cmeq v0.2s, v0.2s, v4.2s +; CHECK-NEXT: cmge v5.2s, v3.2s, #0 ; CHECK-NEXT: cmlt v4.2s, v3.2s, #0 -; CHECK-NEXT: bic v1.8b, v1.8b, v4.8b -; CHECK-NEXT: bic v4.2s, #128, lsl #24 +; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s +; CHECK-NEXT: cmeq v0.2s, v0.2s, v5.2s +; CHECK-NEXT: mvni v1.2s, #128, lsl #24 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v2.8b, v2.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v1.8b, v4.8b, v1.8b +; CHECK-NEXT: bsl v1.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v2.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v1.8b, v3.8b ; CHECK-NEXT: ushr v0.2s, v0.2s, #24 @@ -401,19 +396,18 @@ ; CHECK: // %bb.0: ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] -; CHECK-NEXT: movi v2.4h, #128, lsl #8 +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 ; CHECK-NEXT: sub v3.4h, v0.4h, v1.4h ; CHECK-NEXT: cmge v1.4h, v1.4h, #0 ; CHECK-NEXT: cmge v0.4h, v0.4h, #0 -; CHECK-NEXT: cmge v4.4h, v3.4h, #0 -; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h -; CHECK-NEXT: cmeq v0.4h, v0.4h, v4.4h +; CHECK-NEXT: cmge v5.4h, v3.4h, #0 ; CHECK-NEXT: cmlt v4.4h, v3.4h, #0 -; CHECK-NEXT: bic v2.8b, v2.8b, v4.8b -; CHECK-NEXT: bic v4.4h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h +; CHECK-NEXT: cmeq v0.4h, v0.4h, v5.4h +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v1.8b, v1.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v2.8b, v4.8b, v2.8b +; CHECK-NEXT: bsl v2.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b ; CHECK-NEXT: str d0, [x2] @@ -441,16 +435,15 @@ ; CHECK-NEXT: sub v3.2s, v0.2s, v2.2s ; CHECK-NEXT: cmge v2.2s, v2.2s, #0 ; CHECK-NEXT: cmge v0.2s, v0.2s, #0 -; CHECK-NEXT: cmge v4.2s, v3.2s, #0 -; CHECK-NEXT: movi v1.2s, #128, lsl #24 -; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s -; CHECK-NEXT: cmeq v0.2s, v0.2s, v4.2s +; CHECK-NEXT: cmge v5.2s, v3.2s, #0 ; CHECK-NEXT: cmlt v4.2s, v3.2s, #0 -; CHECK-NEXT: bic v1.8b, v1.8b, v4.8b -; CHECK-NEXT: bic v4.2s, #128, lsl #24 +; CHECK-NEXT: cmeq v2.2s, v0.2s, v2.2s +; CHECK-NEXT: cmeq v0.2s, v0.2s, v5.2s +; CHECK-NEXT: mvni v1.2s, #128, lsl #24 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v2.8b, v2.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v1.8b, v4.8b, v1.8b +; CHECK-NEXT: bsl v1.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v2.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v1.8b, v3.8b ; CHECK-NEXT: ushr v0.2s, v0.2s, #16 @@ -491,39 +484,38 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind { ; CHECK-LABEL: v12i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q2, q0, [x0] -; CHECK-NEXT: ldp q3, q1, [x1] -; CHECK-NEXT: movi v4.8h, #128, lsl #8 -; CHECK-NEXT: sub v5.8h, v0.8h, v1.8h +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ldp q3, q2, [x1] +; CHECK-NEXT: mvni v5.8h, #128, lsl #8 +; CHECK-NEXT: mvni v4.8h, #128, lsl #8 +; CHECK-NEXT: sub v6.8h, v1.8h, v2.8h +; CHECK-NEXT: cmlt v16.8h, v6.8h, #0 +; CHECK-NEXT: sub v7.8h, v0.8h, v3.8h +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v17.16b +; CHECK-NEXT: cmlt v16.8h, v7.8h, #0 +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v4.16b, v16.16b, v17.16b +; CHECK-NEXT: cmge v2.8h, v2.8h, #0 ; CHECK-NEXT: cmge v1.8h, v1.8h, #0 -; CHECK-NEXT: cmge v0.8h, v0.8h, #0 -; CHECK-NEXT: sub v6.8h, v2.8h, v3.8h -; CHECK-NEXT: cmge v16.8h, v5.8h, #0 +; CHECK-NEXT: cmge v16.8h, v6.8h, #0 ; CHECK-NEXT: cmge v3.8h, v3.8h, #0 -; CHECK-NEXT: cmge v2.8h, v2.8h, #0 -; CHECK-NEXT: cmlt v7.8h, v5.8h, #0 -; CHECK-NEXT: cmeq v1.8h, v0.8h, v1.8h +; CHECK-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-NEXT: cmeq v2.8h, v1.8h, v2.8h +; CHECK-NEXT: cmeq v1.8h, v1.8h, v16.8h +; CHECK-NEXT: cmge v16.8h, v7.8h, #0 +; CHECK-NEXT: cmeq v3.8h, v0.8h, v3.8h ; CHECK-NEXT: cmeq v0.8h, v0.8h, v16.8h -; CHECK-NEXT: cmge v16.8h, v6.8h, #0 -; CHECK-NEXT: cmeq v3.8h, v2.8h, v3.8h -; CHECK-NEXT: cmeq v2.8h, v2.8h, v16.8h -; CHECK-NEXT: bic v16.16b, v4.16b, v7.16b -; CHECK-NEXT: bic v7.8h, #128, lsl #8 -; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b -; CHECK-NEXT: cmlt v16.8h, v6.8h, #0 -; CHECK-NEXT: bic v4.16b, v4.16b, v16.16b -; CHECK-NEXT: bic v16.8h, #128, lsl #8 -; CHECK-NEXT: mvn v1.16b, v1.16b +; CHECK-NEXT: mvn v2.16b, v2.16b ; CHECK-NEXT: mvn v3.16b, v3.16b +; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: mvn v2.16b, v2.16b -; CHECK-NEXT: orr v4.16b, v16.16b, v4.16b -; CHECK-NEXT: and v0.16b, v1.16b, v0.16b -; CHECK-NEXT: and v1.16b, v3.16b, v2.16b -; CHECK-NEXT: bsl v0.16b, v7.16b, v5.16b -; CHECK-NEXT: bsl v1.16b, v4.16b, v6.16b -; CHECK-NEXT: str q1, [x2] -; CHECK-NEXT: str d0, [x2, #16] +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: and v0.16b, v3.16b, v0.16b +; CHECK-NEXT: bsl v1.16b, v5.16b, v6.16b +; CHECK-NEXT: bsl v0.16b, v4.16b, v7.16b +; CHECK-NEXT: str q0, [x2] +; CHECK-NEXT: str d1, [x2, #16] ; CHECK-NEXT: ret %x = load <12 x i16>, <12 x i16>* %px %y = load <12 x i16>, <12 x i16>* %py @@ -565,19 +557,18 @@ ; CHECK: // %bb.0: ; CHECK-NEXT: ldr h0, [x0] ; CHECK-NEXT: ldr h1, [x1] -; CHECK-NEXT: movi v2.4h, #128, lsl #8 +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 ; CHECK-NEXT: sub v3.4h, v0.4h, v1.4h ; CHECK-NEXT: cmge v1.4h, v1.4h, #0 ; CHECK-NEXT: cmge v0.4h, v0.4h, #0 -; CHECK-NEXT: cmge v4.4h, v3.4h, #0 -; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h -; CHECK-NEXT: cmeq v0.4h, v0.4h, v4.4h +; CHECK-NEXT: cmge v5.4h, v3.4h, #0 ; CHECK-NEXT: cmlt v4.4h, v3.4h, #0 -; CHECK-NEXT: bic v2.8b, v2.8b, v4.8b -; CHECK-NEXT: bic v4.4h, #128, lsl #8 +; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h +; CHECK-NEXT: cmeq v0.4h, v0.4h, v5.4h +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v1.8b, v1.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v2.8b, v4.8b, v2.8b +; CHECK-NEXT: bsl v2.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v2.8b, v3.8b ; CHECK-NEXT: str h0, [x2] @@ -645,16 +636,15 @@ ; CHECK-NEXT: sub v2.2s, v0.2s, v1.2s ; CHECK-NEXT: cmge v1.2s, v1.2s, #0 ; CHECK-NEXT: cmge v0.2s, v0.2s, #0 -; CHECK-NEXT: cmge v4.2s, v2.2s, #0 -; CHECK-NEXT: movi v3.2s, #128, lsl #24 -; CHECK-NEXT: cmeq v1.2s, v0.2s, v1.2s -; CHECK-NEXT: cmeq v0.2s, v0.2s, v4.2s +; CHECK-NEXT: cmge v5.2s, v2.2s, #0 ; CHECK-NEXT: cmlt v4.2s, v2.2s, #0 -; CHECK-NEXT: bic v3.8b, v3.8b, v4.8b -; CHECK-NEXT: bic v4.2s, #128, lsl #24 +; CHECK-NEXT: cmeq v1.2s, v0.2s, v1.2s +; CHECK-NEXT: cmeq v0.2s, v0.2s, v5.2s +; CHECK-NEXT: mvni v3.2s, #128, lsl #24 +; CHECK-NEXT: mvn v5.8b, v4.8b ; CHECK-NEXT: mvn v1.8b, v1.8b ; CHECK-NEXT: mvn v0.8b, v0.8b -; CHECK-NEXT: orr v3.8b, v4.8b, v3.8b +; CHECK-NEXT: bsl v3.8b, v4.8b, v5.8b ; CHECK-NEXT: and v0.8b, v1.8b, v0.8b ; CHECK-NEXT: bsl v0.8b, v3.8b, v2.8b ; CHECK-NEXT: ret @@ -668,16 +658,15 @@ ; CHECK-NEXT: sub v2.4s, v0.4s, v1.4s ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 ; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: cmge v4.4s, v2.4s, #0 -; CHECK-NEXT: movi v3.4s, #128, lsl #24 -; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s -; CHECK-NEXT: cmeq v0.4s, v0.4s, v4.4s +; CHECK-NEXT: cmge v5.4s, v2.4s, #0 ; CHECK-NEXT: cmlt v4.4s, v2.4s, #0 -; CHECK-NEXT: bic v3.16b, v3.16b, v4.16b -; CHECK-NEXT: bic v4.4s, #128, lsl #24 +; CHECK-NEXT: cmeq v1.4s, v0.4s, v1.4s +; CHECK-NEXT: cmeq v0.4s, v0.4s, v5.4s +; CHECK-NEXT: mvni v3.4s, #128, lsl #24 +; CHECK-NEXT: mvn v5.16b, v4.16b ; CHECK-NEXT: mvn v1.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v3.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v3.16b, v4.16b, v5.16b ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b ; CHECK-NEXT: bsl v0.16b, v3.16b, v2.16b ; CHECK-NEXT: ret @@ -689,34 +678,33 @@ ; CHECK-LABEL: v8i32: ; CHECK: // %bb.0: ; CHECK-NEXT: sub v4.4s, v0.4s, v2.4s +; CHECK-NEXT: cmlt v16.4s, v4.4s, #0 +; CHECK-NEXT: mvni v6.4s, #128, lsl #24 +; CHECK-NEXT: sub v7.4s, v1.4s, v3.4s +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v6.16b, v16.16b, v17.16b +; CHECK-NEXT: cmlt v16.4s, v7.4s, #0 +; CHECK-NEXT: mvni v5.4s, #128, lsl #24 +; CHECK-NEXT: mvn v17.16b, v16.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v17.16b ; CHECK-NEXT: cmge v2.4s, v2.4s, #0 ; CHECK-NEXT: cmge v0.4s, v0.4s, #0 -; CHECK-NEXT: sub v6.4s, v1.4s, v3.4s ; CHECK-NEXT: cmge v16.4s, v4.4s, #0 -; CHECK-NEXT: movi v5.4s, #128, lsl #24 ; CHECK-NEXT: cmge v3.4s, v3.4s, #0 ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 -; CHECK-NEXT: cmlt v7.4s, v4.4s, #0 ; CHECK-NEXT: cmeq v2.4s, v0.4s, v2.4s ; CHECK-NEXT: cmeq v0.4s, v0.4s, v16.4s -; CHECK-NEXT: cmge v16.4s, v6.4s, #0 +; CHECK-NEXT: cmge v16.4s, v7.4s, #0 ; CHECK-NEXT: cmeq v3.4s, v1.4s, v3.4s ; CHECK-NEXT: cmeq v1.4s, v1.4s, v16.4s -; CHECK-NEXT: bic v16.16b, v5.16b, v7.16b -; CHECK-NEXT: bic v7.4s, #128, lsl #24 -; CHECK-NEXT: orr v7.16b, v7.16b, v16.16b -; CHECK-NEXT: cmlt v16.4s, v6.4s, #0 -; CHECK-NEXT: bic v5.16b, v5.16b, v16.16b -; CHECK-NEXT: bic v16.4s, #128, lsl #24 ; CHECK-NEXT: mvn v2.16b, v2.16b ; CHECK-NEXT: mvn v3.16b, v3.16b ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: orr v5.16b, v16.16b, v5.16b ; CHECK-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-NEXT: and v1.16b, v3.16b, v1.16b -; CHECK-NEXT: bsl v0.16b, v7.16b, v4.16b -; CHECK-NEXT: bsl v1.16b, v5.16b, v6.16b +; CHECK-NEXT: bsl v0.16b, v6.16b, v4.16b +; CHECK-NEXT: bsl v1.16b, v5.16b, v7.16b ; CHECK-NEXT: ret %z = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %x, <8 x i32> %y) ret <8 x i32> %z @@ -725,63 +713,62 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind { ; CHECK-LABEL: v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v18.4s, v1.4s, v5.4s +; CHECK-NEXT: sub v16.4s, v0.4s, v4.4s +; CHECK-NEXT: cmlt v24.4s, v16.4s, #0 +; CHECK-NEXT: mvni v18.4s, #128, lsl #24 +; CHECK-NEXT: sub v19.4s, v1.4s, v5.4s +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v18.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.4s, v19.4s, #0 +; CHECK-NEXT: mvni v20.4s, #128, lsl #24 +; CHECK-NEXT: sub v21.4s, v2.4s, v6.4s +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v20.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.4s, v21.4s, #0 +; CHECK-NEXT: mvni v22.4s, #128, lsl #24 +; CHECK-NEXT: sub v23.4s, v3.4s, v7.4s +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v22.16b, v24.16b, v25.16b +; CHECK-NEXT: cmlt v24.4s, v23.4s, #0 +; CHECK-NEXT: mvni v17.4s, #128, lsl #24 +; CHECK-NEXT: mvn v25.16b, v24.16b +; CHECK-NEXT: bsl v17.16b, v24.16b, v25.16b +; CHECK-NEXT: cmge v4.4s, v4.4s, #0 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: cmge v24.4s, v16.4s, #0 ; CHECK-NEXT: cmge v5.4s, v5.4s, #0 ; CHECK-NEXT: cmge v1.4s, v1.4s, #0 -; CHECK-NEXT: sub v19.4s, v2.4s, v6.4s -; CHECK-NEXT: cmge v24.4s, v18.4s, #0 -; CHECK-NEXT: sub v16.4s, v0.4s, v4.4s +; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s +; CHECK-NEXT: cmeq v0.4s, v0.4s, v24.4s +; CHECK-NEXT: cmge v24.4s, v19.4s, #0 ; CHECK-NEXT: cmge v6.4s, v6.4s, #0 ; CHECK-NEXT: cmge v2.4s, v2.4s, #0 -; CHECK-NEXT: sub v20.4s, v3.4s, v7.4s ; CHECK-NEXT: cmeq v5.4s, v1.4s, v5.4s ; CHECK-NEXT: cmeq v1.4s, v1.4s, v24.4s -; CHECK-NEXT: cmge v24.4s, v19.4s, #0 -; CHECK-NEXT: movi v17.4s, #128, lsl #24 -; CHECK-NEXT: cmge v4.4s, v4.4s, #0 -; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: cmge v24.4s, v21.4s, #0 +; CHECK-NEXT: mvn v4.16b, v4.16b +; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: cmge v7.4s, v7.4s, #0 ; CHECK-NEXT: cmge v3.4s, v3.4s, #0 -; CHECK-NEXT: cmlt v21.4s, v16.4s, #0 -; CHECK-NEXT: cmge v22.4s, v16.4s, #0 ; CHECK-NEXT: cmeq v6.4s, v2.4s, v6.4s ; CHECK-NEXT: cmeq v2.4s, v2.4s, v24.4s -; CHECK-NEXT: cmge v24.4s, v20.4s, #0 -; CHECK-NEXT: cmeq v4.4s, v0.4s, v4.4s -; CHECK-NEXT: cmlt v23.4s, v18.4s, #0 -; CHECK-NEXT: cmeq v0.4s, v0.4s, v22.4s -; CHECK-NEXT: cmeq v7.4s, v3.4s, v7.4s -; CHECK-NEXT: cmeq v3.4s, v3.4s, v24.4s -; CHECK-NEXT: bic v24.16b, v17.16b, v21.16b -; CHECK-NEXT: bic v21.4s, #128, lsl #24 -; CHECK-NEXT: cmlt v22.4s, v19.4s, #0 -; CHECK-NEXT: orr v21.16b, v21.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v23.16b -; CHECK-NEXT: bic v23.4s, #128, lsl #24 -; CHECK-NEXT: mvn v4.16b, v4.16b -; CHECK-NEXT: mvn v0.16b, v0.16b -; CHECK-NEXT: orr v23.16b, v23.16b, v24.16b -; CHECK-NEXT: bic v24.16b, v17.16b, v22.16b -; CHECK-NEXT: bic v22.4s, #128, lsl #24 +; CHECK-NEXT: cmge v24.4s, v23.4s, #0 ; CHECK-NEXT: and v0.16b, v4.16b, v0.16b ; CHECK-NEXT: mvn v4.16b, v5.16b ; CHECK-NEXT: mvn v1.16b, v1.16b -; CHECK-NEXT: orr v22.16b, v22.16b, v24.16b -; CHECK-NEXT: cmlt v24.4s, v20.4s, #0 +; CHECK-NEXT: cmeq v7.4s, v3.4s, v7.4s +; CHECK-NEXT: cmeq v3.4s, v3.4s, v24.4s ; CHECK-NEXT: and v1.16b, v4.16b, v1.16b ; CHECK-NEXT: mvn v4.16b, v6.16b ; CHECK-NEXT: mvn v2.16b, v2.16b -; CHECK-NEXT: bic v17.16b, v17.16b, v24.16b -; CHECK-NEXT: bic v24.4s, #128, lsl #24 ; CHECK-NEXT: and v2.16b, v4.16b, v2.16b ; CHECK-NEXT: mvn v4.16b, v7.16b ; CHECK-NEXT: mvn v3.16b, v3.16b -; CHECK-NEXT: orr v17.16b, v24.16b, v17.16b ; CHECK-NEXT: and v3.16b, v4.16b, v3.16b -; CHECK-NEXT: bsl v0.16b, v21.16b, v16.16b -; CHECK-NEXT: bsl v1.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v2.16b, v22.16b, v19.16b -; CHECK-NEXT: bsl v3.16b, v17.16b, v20.16b +; CHECK-NEXT: bsl v0.16b, v18.16b, v16.16b +; CHECK-NEXT: bsl v1.16b, v20.16b, v19.16b +; CHECK-NEXT: bsl v2.16b, v22.16b, v21.16b +; CHECK-NEXT: bsl v3.16b, v17.16b, v23.16b ; CHECK-NEXT: ret %z = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %x, <16 x i32> %y) ret <16 x i32> %z Index: llvm/test/CodeGen/AArch64/uadd_sat_vec.ll =================================================================== --- llvm/test/CodeGen/AArch64/uadd_sat_vec.ll +++ llvm/test/CodeGen/AArch64/uadd_sat_vec.ll @@ -405,7 +405,6 @@ ; CHECK-NEXT: add v1.2d, v0.2d, v1.2d ; CHECK-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret %z = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y) @@ -420,9 +419,7 @@ ; CHECK-NEXT: cmhi v0.2d, v0.2d, v2.2d ; CHECK-NEXT: cmhi v1.2d, v1.2d, v3.2d ; CHECK-NEXT: bic v2.16b, v2.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: bic v3.16b, v3.16b, v1.16b -; CHECK-NEXT: bic v1.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-NEXT: ret @@ -442,13 +439,9 @@ ; CHECK-NEXT: cmhi v2.2d, v2.2d, v6.2d ; CHECK-NEXT: cmhi v3.2d, v3.2d, v7.2d ; CHECK-NEXT: bic v4.16b, v4.16b, v0.16b -; CHECK-NEXT: bic v0.4s, #0 ; CHECK-NEXT: bic v5.16b, v5.16b, v1.16b -; CHECK-NEXT: bic v1.4s, #0 ; CHECK-NEXT: bic v6.16b, v6.16b, v2.16b -; CHECK-NEXT: bic v2.4s, #0 ; CHECK-NEXT: bic v7.16b, v7.16b, v3.16b -; CHECK-NEXT: bic v3.4s, #0 ; CHECK-NEXT: orr v0.16b, v0.16b, v4.16b ; CHECK-NEXT: orr v1.16b, v1.16b, v5.16b ; CHECK-NEXT: orr v2.16b, v2.16b, v6.16b