Index: llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -26489,10 +26489,18 @@ // select_cc seteq X, 0, sizeof(X), ctlz_zero_undef(X) -> ctlz(X) // select_cc seteq X, 0, sizeof(X), cttz(X) -> cttz(X) // select_cc seteq X, 0, sizeof(X), cttz_zero_undef(X) -> cttz(X) + // select_cc seteq X, 0, 0, cttz -> and(cttz(X), sizeof(X) - 1) + // select_cc seteq X, 0, 0, cttz_zero_undef(X) -> and(cttz(X), sizeof(X) - 1) + // select_cc seteq X, 0, 0, ctlz -> and(ctlz(X), sizeof(X) - 1) + // select_cc seteq X, 0, 0, ctlz_zero_undef(X) -> and(ctlz(X), sizeof(X) - 1) // select_cc setne X, 0, ctlz(X), sizeof(X) -> ctlz(X) // select_cc setne X, 0, ctlz_zero_undef(X), sizeof(X) -> ctlz(X) // select_cc setne X, 0, cttz(X), sizeof(X) -> cttz(X) // select_cc setne X, 0, cttz_zero_undef(X), sizeof(X) -> cttz(X) + // select_cc setne X, 0, cttz, 0 -> and(cttz(X), sizeof(X) - 1) + // select_cc setne X, 0, cttz_zero_undef(X), 0 -> and(cttz(X), sizeof(X) - 1) + // select_cc setne X, 0, ctlz, 0 -> and(ctlz(X), sizeof(X) - 1) + // select_cc setne X, 0, ctlz_zero_undef(X), 0 -> and(ctlz(X), sizeof(X) - 1) if (N1C && N1C->isZero() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { SDValue ValueOnZero = N2; SDValue Count = N3; @@ -26501,21 +26509,32 @@ std::swap(ValueOnZero, Count); // Check if the value on zero is a constant equal to the bits in the type. if (auto *ValueOnZeroC = dyn_cast(ValueOnZero)) { - if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) { + bool ZeroWhenZero = ValueOnZeroC->getAPIntValue().isZero(); + unsigned VTSize = VT.getSizeInBits(); + SDValue Res; + if ((ZeroWhenZero && isPowerOf2_64(VTSize)) || + ValueOnZeroC->getAPIntValue() == VTSize) { // If the other operand is cttz/cttz_zero_undef of N0, and cttz is // legal, combine to just cttz. if ((Count.getOpcode() == ISD::CTTZ || Count.getOpcode() == ISD::CTTZ_ZERO_UNDEF) && N0 == Count.getOperand(0) && (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) - return DAG.getNode(ISD::CTTZ, DL, VT, N0); + Res = DAG.getNode(ISD::CTTZ, DL, VT, N0); // If the other operand is ctlz/ctlz_zero_undef of N0, and ctlz is // legal, combine to just ctlz. if ((Count.getOpcode() == ISD::CTLZ || Count.getOpcode() == ISD::CTLZ_ZERO_UNDEF) && N0 == Count.getOperand(0) && (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) - return DAG.getNode(ISD::CTLZ, DL, VT, N0); + Res = DAG.getNode(ISD::CTLZ, DL, VT, N0); + + if (Res) { + if (ZeroWhenZero) + return DAG.getNode(ISD::AND, DL, VT, Res, + DAG.getConstant(VTSize - 1, DL, VT)); + return Res; + } } } } Index: llvm/test/CodeGen/AArch64/fold-csel-cttz-and.ll =================================================================== --- llvm/test/CodeGen/AArch64/fold-csel-cttz-and.ll +++ llvm/test/CodeGen/AArch64/fold-csel-cttz-and.ll @@ -127,9 +127,9 @@ define i32 @notcttz(i32 %x) { ; CHECK-LABEL: notcttz: ; CHECK: // %bb.0: // %entry + ; CHECK-NEXT: clz w8, w0 -; CHECK-NEXT: cmp w0, #0 -; CHECK-NEXT: csel w0, wzr, w8, eq +; CHECK-NEXT: and w0, w8, #0x1f ; CHECK-NEXT: ret entry: %0 = call i32 @llvm.ctlz.i32(i32 %x, i1 true)