Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -1629,8 +1629,8 @@ /// Returns true if \p V is an FP constant with a value of positive zero. bool isNullFPConstant(SDValue V); -/// Returns true if \p V is an integer constant with all bits set. -bool isAllOnesConstant(SDValue V); +/// Returns true if \p V is an integer constant with all bits in VT set, +bool isAllOnesConstant(SDValue V, MVT VT = MVT()); /// Returns true if \p V is a constant integer one. bool isOneConstant(SDValue V); Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8612,9 +8612,15 @@ return Const != nullptr && Const->isZero() && !Const->isNegative(); } -bool llvm::isAllOnesConstant(SDValue V) { +bool llvm::isAllOnesConstant(SDValue V, MVT VT) { ConstantSDNode *Const = dyn_cast(V); - return Const != nullptr && Const->isAllOnesValue(); + if (!Const) + return false; + if (!VT.isValid()) + return Const->isAllOnesValue(); + return Const->getAPIntValue() + .zextOrTrunc(VT.getSizeInBits()) + .isAllOnesValue(); } bool llvm::isOneConstant(SDValue V) { Index: lib/Target/X86/X86ISelDAGToDAG.cpp =================================================================== --- lib/Target/X86/X86ISelDAGToDAG.cpp +++ lib/Target/X86/X86ISelDAGToDAG.cpp @@ -3022,15 +3022,21 @@ }; // b) x & ~(-1 << nbits) - auto matchPatternB = [&checkOneUse, &NBits](SDValue Mask) -> bool { + auto matchPatternB = [&checkOneUse, &peekThroughOneUseTruncation, NVT, + &NBits](SDValue Mask) -> bool { // Match `~()`. Must only have one use! - if (!isBitwiseNot(Mask) || !checkOneUse(Mask)) + if (Mask.getOpcode() != ISD::XOR || !checkOneUse(Mask)) return false; - // Match `-1 << nbits`. Must only have one use! - SDValue M0 = Mask->getOperand(0); + // The -1 only has to be all-ones for the final Node's NVT. + if (!isAllOnesConstant(peekThroughOneUseTruncation(Mask->getOperand(1)), + NVT)) + return false; + // Match `-1 << nbits`. Might be truncated. Must only have one use! + SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0)); if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0)) return false; - if (!isAllOnesConstant(M0->getOperand(0))) + // The -1 only has to be all-ones for the final Node's NVT. + if (!isAllOnesConstant(M0->getOperand(0), NVT)) return false; NBits = M0->getOperand(1); return true; Index: test/CodeGen/X86/extract-bits.ll =================================================================== --- test/CodeGen/X86/extract-bits.ll +++ test/CodeGen/X86/extract-bits.ll @@ -3667,22 +3667,17 @@ ; ; X64-BMI1NOTBM-LABEL: bextr64_32_b0: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movq %rsi, %rcx -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $rcx -; X64-BMI1NOTBM-NEXT: shrq %cl, %rdi -; X64-BMI1NOTBM-NEXT: movq $-1, %rax -; X64-BMI1NOTBM-NEXT: movl %edx, %ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rax -; X64-BMI1NOTBM-NEXT: andnl %edi, %eax, %eax +; X64-BMI1NOTBM-NEXT: shll $8, %edx +; X64-BMI1NOTBM-NEXT: movzbl %sil, %eax +; X64-BMI1NOTBM-NEXT: orl %edx, %eax +; X64-BMI1NOTBM-NEXT: bextrq %rax, %rdi, %rax +; X64-BMI1NOTBM-NEXT: # kill: def $eax killed $eax killed $rax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bextr64_32_b0: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $edx killed $edx def $rdx ; X64-BMI1BMI2-NEXT: shrxq %rsi, %rdi, %rax -; X64-BMI1BMI2-NEXT: movq $-1, %rcx -; X64-BMI1BMI2-NEXT: shlxq %rdx, %rcx, %rcx -; X64-BMI1BMI2-NEXT: andnl %eax, %ecx, %eax +; X64-BMI1BMI2-NEXT: bzhil %edx, %eax, %eax ; X64-BMI1BMI2-NEXT: retq %shiftedval = lshr i64 %val, %numskipbits %widenumlowbits = zext i8 %numlowbits to i64 @@ -4001,27 +3996,17 @@ ; ; X64-BMI1NOTBM-LABEL: bextr64_32_b3: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movq %rsi, %rcx -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $rcx -; X64-BMI1NOTBM-NEXT: shrq %cl, %rdi -; X64-BMI1NOTBM-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: movl $4294967295, %esi # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: movl %edx, %ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rsi -; X64-BMI1NOTBM-NEXT: xorl %esi, %eax -; X64-BMI1NOTBM-NEXT: andl %edi, %eax +; X64-BMI1NOTBM-NEXT: shll $8, %edx +; X64-BMI1NOTBM-NEXT: movzbl %sil, %eax +; X64-BMI1NOTBM-NEXT: orl %edx, %eax +; X64-BMI1NOTBM-NEXT: bextrq %rax, %rdi, %rax ; X64-BMI1NOTBM-NEXT: # kill: def $eax killed $eax killed $rax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bextr64_32_b3: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $edx killed $edx def $rdx ; X64-BMI1BMI2-NEXT: shrxq %rsi, %rdi, %rax -; X64-BMI1BMI2-NEXT: movl $4294967295, %ecx # imm = 0xFFFFFFFF -; X64-BMI1BMI2-NEXT: shlxq %rdx, %rcx, %rdx -; X64-BMI1BMI2-NEXT: xorl %edx, %ecx -; X64-BMI1BMI2-NEXT: andl %ecx, %eax -; X64-BMI1BMI2-NEXT: # kill: def $eax killed $eax killed $rax +; X64-BMI1BMI2-NEXT: bzhil %edx, %eax, %eax ; X64-BMI1BMI2-NEXT: retq %shiftedval = lshr i64 %val, %numskipbits %widenumlowbits = zext i8 %numlowbits to i64 Index: test/CodeGen/X86/extract-lowbits.ll =================================================================== --- test/CodeGen/X86/extract-lowbits.ll +++ test/CodeGen/X86/extract-lowbits.ll @@ -1845,19 +1845,13 @@ ; ; X64-BMI1NOTBM-LABEL: bzhi64_32_b0: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movl %esi, %ecx -; X64-BMI1NOTBM-NEXT: movq $-1, %rax -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rax -; X64-BMI1NOTBM-NEXT: andnl %edi, %eax, %eax +; X64-BMI1NOTBM-NEXT: shll $8, %esi +; X64-BMI1NOTBM-NEXT: bextrl %esi, %edi, %eax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bzhi64_32_b0: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $esi killed $esi def $rsi -; X64-BMI1BMI2-NEXT: movq $-1, %rax -; X64-BMI1BMI2-NEXT: shlxq %rsi, %rax, %rax -; X64-BMI1BMI2-NEXT: andnl %edi, %eax, %eax +; X64-BMI1BMI2-NEXT: bzhil %esi, %edi, %eax ; X64-BMI1BMI2-NEXT: retq %widenumlowbits = zext i8 %numlowbits to i64 %notmask = shl nsw i64 -1, %widenumlowbits @@ -2032,24 +2026,13 @@ ; ; X64-BMI1NOTBM-LABEL: bzhi64_32_b3: ; X64-BMI1NOTBM: # %bb.0: -; X64-BMI1NOTBM-NEXT: movl %esi, %ecx -; X64-BMI1NOTBM-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: movl $4294967295, %edx # imm = 0xFFFFFFFF -; X64-BMI1NOTBM-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-BMI1NOTBM-NEXT: shlq %cl, %rdx -; X64-BMI1NOTBM-NEXT: xorl %edx, %eax -; X64-BMI1NOTBM-NEXT: andl %edi, %eax -; X64-BMI1NOTBM-NEXT: # kill: def $eax killed $eax killed $rax +; X64-BMI1NOTBM-NEXT: shll $8, %esi +; X64-BMI1NOTBM-NEXT: bextrl %esi, %edi, %eax ; X64-BMI1NOTBM-NEXT: retq ; ; X64-BMI1BMI2-LABEL: bzhi64_32_b3: ; X64-BMI1BMI2: # %bb.0: -; X64-BMI1BMI2-NEXT: # kill: def $esi killed $esi def $rsi -; X64-BMI1BMI2-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF -; X64-BMI1BMI2-NEXT: shlxq %rsi, %rax, %rcx -; X64-BMI1BMI2-NEXT: xorl %ecx, %eax -; X64-BMI1BMI2-NEXT: andl %edi, %eax -; X64-BMI1BMI2-NEXT: # kill: def $eax killed $eax killed $rax +; X64-BMI1BMI2-NEXT: bzhil %esi, %edi, %eax ; X64-BMI1BMI2-NEXT: retq %widenumlowbits = zext i8 %numlowbits to i64 %notmask = shl nsw i64 4294967295, %widenumlowbits