Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -1468,6 +1468,17 @@ bool MaskedValueIsZero(SDValue Op, const APInt &Mask, const APInt &DemandedElts, unsigned Depth = 0) const; + /// Return true if '(Op & Mask) == Mask'. + /// Op and Mask are known to be the same type. + bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, + unsigned Depth = 0) const; + + /// Return true if '(Op & Mask) == Mask'. + /// Op and Mask are known to be the same type. + bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, + const APInt &DemandedElts, + unsigned Depth = 0) const; + /// Determine which bits of Op are known to be either zero or one and return /// them in Known. For vectors, the known bits are those that are shared by /// every vector element. Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2244,6 +2244,25 @@ return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero); } +/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. +/// Mask is known to be zero for bits that V cannot have. +bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, + unsigned Depth) const { + EVT VT = V.getValueType(); + APInt DemandedElts = VT.isVector() + ? APInt::getAllOnesValue(VT.getVectorNumElements()) + : APInt(1, 1); + return MaskedValueIsAllOnes(V, Mask, DemandedElts, Depth); +} + +/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask' in DemandedElts. +/// Mask is known to be zero for bits that V cannot have. +bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, + const APInt &DemandedElts, + unsigned Depth) const { + return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).One); +} + /// isSplatValue - Return true if the vector V has the same value /// across all DemandedElts. bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, Index: lib/Target/X86/X86ISelDAGToDAG.cpp =================================================================== --- lib/Target/X86/X86ISelDAGToDAG.cpp +++ lib/Target/X86/X86ISelDAGToDAG.cpp @@ -3021,16 +3021,28 @@ return true; }; + auto isAllOnes = [this, peekThroughOneUseTruncation, NVT](SDValue V) { + V = peekThroughOneUseTruncation(V); + return CurDAG->MaskedValueIsAllOnes( + V, APInt::getLowBitsSet(V.getSimpleValueType().getSizeInBits(), + NVT.getSizeInBits())); + }; + // b) x & ~(-1 << nbits) - auto matchPatternB = [&checkOneUse, &NBits](SDValue Mask) -> bool { + auto matchPatternB = [&checkOneUse, &peekThroughOneUseTruncation, isAllOnes, + &NBits](SDValue Mask) -> bool { // Match `~()`. Must only have one use! - if (!isBitwiseNot(Mask) || !checkOneUse(Mask)) + if (Mask.getOpcode() != ISD::XOR || !checkOneUse(Mask)) return false; - // Match `-1 << nbits`. Must only have one use! - SDValue M0 = Mask->getOperand(0); + // The -1 only has to be all-ones for the final Node's NVT. + if (!isAllOnes(Mask->getOperand(1))) + return false; + // Match `-1 << nbits`. Might be truncated. Must only have one use! + SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0)); if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0)) return false; - if (!isAllOnesConstant(M0->getOperand(0))) + // The -1 only has to be all-ones for the final Node's NVT. + if (!isAllOnes(M0->getOperand(0))) return false; NBits = M0->getOperand(1); return true; Index: test/CodeGen/X86/extract-bits.ll =================================================================== --- test/CodeGen/X86/extract-bits.ll +++ test/CodeGen/X86/extract-bits.ll @@ -3667,22 +3667,17 @@ ; ; X64-BMI1NOTBM-LABEL: bextr64_32_b0: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movq %rsi, %rcx -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $rcx -; X64-BMI1NOTBM-NEXT: shrq %cl, %rdi -; X64-BMI1NOTBM-NEXT: movq $-1, %rax -; X64-BMI1NOTBM-NEXT: movl %edx, %ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rax -; X64-BMI1NOTBM-NEXT: andnl %edi, %eax, %eax +; X64-BMI1NOTBM-NEXT: shll $8, %edx +; X64-BMI1NOTBM-NEXT: movzbl %sil, %eax +; X64-BMI1NOTBM-NEXT: orl %edx, %eax +; X64-BMI1NOTBM-NEXT: bextrq %rax, %rdi, %rax +; X64-BMI1NOTBM-NEXT: # kill: def $eax killed $eax killed $rax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bextr64_32_b0: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $edx killed $edx def $rdx ; X64-BMI1BMI2-NEXT: shrxq %rsi, %rdi, %rax -; X64-BMI1BMI2-NEXT: movq $-1, %rcx -; X64-BMI1BMI2-NEXT: shlxq %rdx, %rcx, %rcx -; X64-BMI1BMI2-NEXT: andnl %eax, %ecx, %eax +; X64-BMI1BMI2-NEXT: bzhil %edx, %eax, %eax ; X64-BMI1BMI2-NEXT: retq %shiftedval = lshr i64 %val, %numskipbits %widenumlowbits = zext i8 %numlowbits to i64 @@ -4001,27 +3996,17 @@ ; ; X64-BMI1NOTBM-LABEL: bextr64_32_b3: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movq %rsi, %rcx -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $rcx -; X64-BMI1NOTBM-NEXT: shrq %cl, %rdi -; X64-BMI1NOTBM-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: movl $4294967295, %esi # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: movl %edx, %ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rsi -; X64-BMI1NOTBM-NEXT: xorl %esi, %eax -; X64-BMI1NOTBM-NEXT: andl %edi, %eax +; X64-BMI1NOTBM-NEXT: shll $8, %edx +; X64-BMI1NOTBM-NEXT: movzbl %sil, %eax +; X64-BMI1NOTBM-NEXT: orl %edx, %eax +; X64-BMI1NOTBM-NEXT: bextrq %rax, %rdi, %rax ; X64-BMI1NOTBM-NEXT: # kill: def $eax killed $eax killed $rax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bextr64_32_b3: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $edx killed $edx def $rdx ; X64-BMI1BMI2-NEXT: shrxq %rsi, %rdi, %rax -; X64-BMI1BMI2-NEXT: movl $4294967295, %ecx # imm = 0xFFFFFFFF -; X64-BMI1BMI2-NEXT: shlxq %rdx, %rcx, %rdx -; X64-BMI1BMI2-NEXT: xorl %edx, %ecx -; X64-BMI1BMI2-NEXT: andl %ecx, %eax -; X64-BMI1BMI2-NEXT: # kill: def $eax killed $eax killed $rax +; X64-BMI1BMI2-NEXT: bzhil %edx, %eax, %eax ; X64-BMI1BMI2-NEXT: retq %shiftedval = lshr i64 %val, %numskipbits %widenumlowbits = zext i8 %numlowbits to i64 Index: test/CodeGen/X86/extract-lowbits.ll =================================================================== --- test/CodeGen/X86/extract-lowbits.ll +++ test/CodeGen/X86/extract-lowbits.ll @@ -1845,19 +1845,13 @@ ; ; X64-BMI1NOTBM-LABEL: bzhi64_32_b0: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movl %esi, %ecx -; X64-BMI1NOTBM-NEXT: movq $-1, %rax -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rax -; X64-BMI1NOTBM-NEXT: andnl %edi, %eax, %eax +; X64-BMI1NOTBM-NEXT: shll $8, %esi +; X64-BMI1NOTBM-NEXT: bextrl %esi, %edi, %eax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bzhi64_32_b0: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $esi killed $esi def $rsi -; X64-BMI1BMI2-NEXT: movq $-1, %rax -; X64-BMI1BMI2-NEXT: shlxq %rsi, %rax, %rax -; X64-BMI1BMI2-NEXT: andnl %edi, %eax, %eax +; X64-BMI1BMI2-NEXT: bzhil %esi, %edi, %eax ; X64-BMI1BMI2-NEXT: retq %widenumlowbits = zext i8 %numlowbits to i64 %notmask = shl nsw i64 -1, %widenumlowbits @@ -2032,24 +2026,13 @@ ; ; X64-BMI1NOTBM-LABEL: bzhi64_32_b3: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movl %esi, %ecx -; X64-BMI1NOTBM-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: movl $4294967295, %edx # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rdx -; X64-BMI1NOTBM-NEXT: xorl %edx, %eax -; X64-BMI1NOTBM-NEXT: andl %edi, %eax -; X64-BMI1NOTBM-NEXT: # kill: def $eax killed $eax killed $rax +; X64-BMI1NOTBM-NEXT: shll $8, %esi +; X64-BMI1NOTBM-NEXT: bextrl %esi, %edi, %eax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bzhi64_32_b3: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $esi killed $esi def $rsi -; X64-BMI1BMI2-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF -; X64-BMI1BMI2-NEXT: shlxq %rsi, %rax, %rcx -; X64-BMI1BMI2-NEXT: xorl %ecx, %eax -; X64-BMI1BMI2-NEXT: andl %edi, %eax -; X64-BMI1BMI2-NEXT: # kill: def $eax killed $eax killed $rax +; X64-BMI1BMI2-NEXT: bzhil %esi, %edi, %eax ; X64-BMI1BMI2-NEXT: retq %widenumlowbits = zext i8 %numlowbits to i64 %notmask = shl nsw i64 4294967295, %widenumlowbits