Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -14609,6 +14609,47 @@ DAG.getConstant(0, DL, MVT::i64)); } +/// Perform the scalar expression combine in the form of: +/// b + CSEL (c, 1, cc) => CSINC(b+c, b, cc) +static SDValue performAddCSelIntoCSinc(SDNode *N, SelectionDAG &DAG) { + EVT VT = N->getValueType(0); + if (!VT.isScalarInteger()) + return SDValue(); + + SDValue CSel = N->getOperand(0); + SDValue Copy = N->getOperand(1); + if ((CSel.getOpcode() != AArch64ISD::CSEL) || !CSel.hasOneUse() || + !Copy.hasOneUse() || Copy.getOpcode() != ISD::CopyFromReg) + return SDValue(); + + SDValue CCVal = CSel.getOperand(2); + ConstantSDNode *CC = cast(CCVal); + AArch64CC::CondCode AArch64CC = + static_cast(CC->getZExtValue()); + + // The CSEL should include a const one operand. + ConstantSDNode *CTVal = dyn_cast(CSel.getOperand(0)); + ConstantSDNode *CFVal = dyn_cast(CSel.getOperand(1)); + if (!CTVal || !CFVal || (!CTVal->isOne() && !CFVal->isOne())) + return SDValue(); + + // switch CSEL (1, c, cc) to CSEL (c, 1, !cc) + if (CTVal->isOne() && !CFVal->isOne()) { + std::swap(CTVal, CFVal); + AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC); + } + + assert(CFVal->isOne() && "Unexpected constant value"); + + SDLoc DL(N); + SDValue Reg = Copy->getOperand(1); + SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, Reg, SDValue(CTVal, 0)); + CCVal = DAG.getConstant(AArch64CC, DL, MVT::i32); + SDValue Cmp = CSel.getOperand(3); + + return DAG.getNode(AArch64ISD::CSINC, DL, VT, NewNode, Reg, CCVal, Cmp); +} + // ADD(UDOT(zero, x, y), A) --> UDOT(A, x, y) static SDValue performAddDotCombine(SDNode *N, SelectionDAG &DAG) { EVT VT = N->getValueType(0); @@ -14693,6 +14734,8 @@ return Val; if (SDValue Val = performAddDotCombine(N, DAG)) return Val; + if (SDValue Val = performAddCSelIntoCSinc(N, DAG)) + return Val; return performAddSubLongCombine(N, DCI, DAG); } Index: llvm/test/CodeGen/AArch64/aarch64-isel-csinc-type.ll =================================================================== --- llvm/test/CodeGen/AArch64/aarch64-isel-csinc-type.ll +++ llvm/test/CodeGen/AArch64/aarch64-isel-csinc-type.ll @@ -11,9 +11,8 @@ ; CHECK-LABEL: csinc1: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: tst w0, #0xff -; CHECK-NEXT: mov w8, #3 -; CHECK-NEXT: csinc w8, w8, wzr, ne -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: add w8, w1, #3 +; CHECK-NEXT: csinc w0, w8, w1, ne ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i8 %a, 0 @@ -27,9 +26,8 @@ ; CHECK-LABEL: csinc2: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: tst w0, #0xffff -; CHECK-NEXT: mov w8, #3 -; CHECK-NEXT: csinc w8, w8, wzr, ne -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: add w8, w1, #3 +; CHECK-NEXT: csinc w0, w8, w1, ne ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i16 %a, 0 @@ -43,9 +41,8 @@ ; CHECK-LABEL: csinc3: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: cmp w0, #0 -; CHECK-NEXT: mov w8, #3 -; CHECK-NEXT: csinc w8, w8, wzr, ne -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: add w8, w1, #3 +; CHECK-NEXT: csinc w0, w8, w1, ne ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i32 %a, 0 @@ -59,9 +56,8 @@ ; CHECK-LABEL: csinc4: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: cmp x0, #0 -; CHECK-NEXT: mov w8, #3 -; CHECK-NEXT: csinc x8, x8, xzr, ne -; CHECK-NEXT: add x0, x8, x1 +; CHECK-NEXT: add x8, x1, #3 +; CHECK-NEXT: csinc x0, x8, x1, ne ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i64 %a, 0 Index: llvm/test/CodeGen/AArch64/aarch64-isel-csinc.ll =================================================================== --- llvm/test/CodeGen/AArch64/aarch64-isel-csinc.ll +++ llvm/test/CodeGen/AArch64/aarch64-isel-csinc.ll @@ -11,9 +11,8 @@ ; CHECK-LABEL: csinc1: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: cmp w0, #0 -; CHECK-NEXT: mov w8, #3 -; CHECK-NEXT: csinc w8, w8, wzr, eq -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: add w8, w1, #3 +; CHECK-NEXT: csinc w0, w8, w1, eq ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i32 %a, 0 @@ -27,9 +26,8 @@ ; CHECK-LABEL: csinc2: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: cmp w0, #0 -; CHECK-NEXT: mov w8, #3 -; CHECK-NEXT: csinc w8, w8, wzr, ne -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: add w8, w1, #3 +; CHECK-NEXT: csinc w0, w8, w1, ne ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i32 %a, 0 @@ -42,10 +40,9 @@ define dso_local i32 @csinc3(i32 %a, i32 %b) { ; CHECK-LABEL: csinc3: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sub w8, w1, #3 ; CHECK-NEXT: cmp w0, #0 -; CHECK-NEXT: mov w8, #-3 -; CHECK-NEXT: csinc w8, w8, wzr, ne -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: csinc w0, w8, w1, ne ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i32 %a, 0 @@ -58,10 +55,9 @@ define dso_local i32 @csinc4(i32 %a, i32 %b) { ; CHECK-LABEL: csinc4: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sub w8, w1, #3 ; CHECK-NEXT: cmp w0, #0 -; CHECK-NEXT: mov w8, #-3 -; CHECK-NEXT: csinc w8, w8, wzr, eq -; CHECK-NEXT: add w0, w8, w1 +; CHECK-NEXT: csinc w0, w8, w1, eq ; CHECK-NEXT: ret entry: %tobool.not = icmp eq i32 %a, 0