diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -23512,9 +23512,8 @@ /// Result of 'and' is compared against zero. Change to a BT node if possible. /// Returns the BT node and the condition code needed to use it. -static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, - const SDLoc &dl, SelectionDAG &DAG, - SDValue &X86CC) { +static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, const SDLoc &dl, + SelectionDAG &DAG, X86::CondCode &X86CC) { assert(And.getOpcode() == ISD::AND && "Expected AND node!"); SDValue Op0 = And.getOperand(0); SDValue Op1 = And.getOperand(1); @@ -23592,8 +23591,7 @@ if (Src.getValueType() != BitNo.getValueType()) BitNo = DAG.getNode(ISD::ANY_EXTEND, dl, Src.getValueType(), BitNo); - X86CC = DAG.getTargetConstant(CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B, - dl, MVT::i8); + X86CC = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B; return DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo); } @@ -24299,8 +24297,11 @@ // Lower ((X >>s N) & 1) != 0 to BT(X, N). if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && isNullConstant(Op1) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { - if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CC)) + X86::CondCode X86CondCode; + if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) { + X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8); return BT; + } } // Try to use PTEST/PMOVMSKB for a tree ORs equality compared with 0. @@ -24772,9 +24773,9 @@ // We know the result of AND is compared against zero. Try to match // it to BT. if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { - SDValue BTCC; - if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, BTCC)) { - CC = BTCC; + X86::CondCode X86CondCode; + if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) { + CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8); Cond = BT; AddTest = false; } @@ -52283,6 +52284,7 @@ /// If this is an add or subtract where one operand is produced by a cmp+setcc, /// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB} /// with CMP+{ADC, SBB}. +/// Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}. static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT, SDValue X, SDValue Y, SelectionDAG &DAG) { @@ -52290,11 +52292,20 @@ if (Y.getOpcode() == ISD::ZERO_EXTEND && Y.hasOneUse()) Y = Y.getOperand(0); - if (Y.getOpcode() != X86ISD::SETCC || !Y.hasOneUse()) + if (!Y.hasOneUse()) return SDValue(); - X86::CondCode CC = (X86::CondCode)Y.getConstantOperandVal(0); - SDValue EFLAGS = Y.getOperand(1); + X86::CondCode CC; + SDValue EFLAGS; + if (Y.getOpcode() == X86ISD::SETCC) { + CC = (X86::CondCode)Y.getConstantOperandVal(0); + EFLAGS = Y.getOperand(1); + } else if (Y.getOpcode() == ISD::AND && isOneConstant(Y.getOperand(1))) { + EFLAGS = LowerAndToBT(Y, ISD::SETNE, DL, DAG, CC); + } + + if (!EFLAGS) + return SDValue(); // If X is -1 or 0, then we have an opportunity to avoid constants required in // the general case below. diff --git a/llvm/test/CodeGen/X86/add-sub-bool.ll b/llvm/test/CodeGen/X86/add-sub-bool.ll --- a/llvm/test/CodeGen/X86/add-sub-bool.ll +++ b/llvm/test/CodeGen/X86/add-sub-bool.ll @@ -18,31 +18,16 @@ ; X86-LABEL: test_i32_add_add_idx: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: shrl $30, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %ecx, %eax +; X86-NEXT: btl $30, {{[0-9]+}}(%esp) +; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax ; X86-NEXT: retl ; -; NOTBM-LABEL: test_i32_add_add_idx: -; NOTBM: # %bb.0: -; NOTBM-NEXT: # kill: def $esi killed $esi def $rsi -; NOTBM-NEXT: # kill: def $edi killed $edi def $rdi -; NOTBM-NEXT: leal (%rdi,%rsi), %eax -; NOTBM-NEXT: shrl $30, %edx -; NOTBM-NEXT: andl $1, %edx -; NOTBM-NEXT: addl %edx, %eax -; NOTBM-NEXT: retq -; -; TBM-LABEL: test_i32_add_add_idx: -; TBM: # %bb.0: -; TBM-NEXT: # kill: def $esi killed $esi def $rsi -; TBM-NEXT: # kill: def $edi killed $edi def $rdi -; TBM-NEXT: bextrl $286, %edx, %eax # imm = 0x11E -; TBM-NEXT: addl %edi, %eax -; TBM-NEXT: addl %esi, %eax -; TBM-NEXT: retq +; X64-LABEL: test_i32_add_add_idx: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: btl $30, %edx +; X64-NEXT: adcl %esi, %eax +; X64-NEXT: retq %add = add i32 %y, %x %shift = lshr i32 %z, 30 %mask = and i32 %shift, 1 @@ -54,31 +39,16 @@ ; X86-LABEL: test_i32_add_add_commute_idx: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: shrl $2, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %ecx, %eax +; X86-NEXT: btl $2, {{[0-9]+}}(%esp) +; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax ; X86-NEXT: retl ; -; NOTBM-LABEL: test_i32_add_add_commute_idx: -; NOTBM: # %bb.0: -; NOTBM-NEXT: # kill: def $esi killed $esi def $rsi -; NOTBM-NEXT: # kill: def $edi killed $edi def $rdi -; NOTBM-NEXT: leal (%rdi,%rsi), %eax -; NOTBM-NEXT: shrl $2, %edx -; NOTBM-NEXT: andl $1, %edx -; NOTBM-NEXT: addl %edx, %eax -; NOTBM-NEXT: retq -; -; TBM-LABEL: test_i32_add_add_commute_idx: -; TBM: # %bb.0: -; TBM-NEXT: # kill: def $esi killed $esi def $rsi -; TBM-NEXT: # kill: def $edi killed $edi def $rdi -; TBM-NEXT: bextrl $258, %edx, %eax # imm = 0x102 -; TBM-NEXT: addl %edi, %eax -; TBM-NEXT: addl %esi, %eax -; TBM-NEXT: retq +; X64-LABEL: test_i32_add_add_commute_idx: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: btl $2, %edx +; X64-NEXT: adcl %esi, %eax +; X64-NEXT: retq %add = add i32 %y, %x %shift = lshr i32 %z, 2 %mask = and i32 %shift, 1 @@ -139,29 +109,18 @@ ; X86-LABEL: test_i32_add_sub_commute_idx: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: shrl $8, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %ecx, %eax +; X86-NEXT: subl {{[0-9]+}}(%esp), %eax +; X86-NEXT: btl $8, {{[0-9]+}}(%esp) +; X86-NEXT: adcl $0, %eax ; X86-NEXT: retl ; -; NOTBM-LABEL: test_i32_add_sub_commute_idx: -; NOTBM: # %bb.0: -; NOTBM-NEXT: # kill: def $edx killed $edx def $rdx -; NOTBM-NEXT: # kill: def $edi killed $edi def $rdi -; NOTBM-NEXT: subl %esi, %edi -; NOTBM-NEXT: shrl $8, %edx -; NOTBM-NEXT: andl $1, %edx -; NOTBM-NEXT: leal (%rdx,%rdi), %eax -; NOTBM-NEXT: retq -; -; TBM-LABEL: test_i32_add_sub_commute_idx: -; TBM: # %bb.0: -; TBM-NEXT: subl %esi, %edi -; TBM-NEXT: bextrl $264, %edx, %eax # imm = 0x108 -; TBM-NEXT: addl %edi, %eax -; TBM-NEXT: retq +; X64-LABEL: test_i32_add_sub_commute_idx: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: subl %esi, %eax +; X64-NEXT: btl $8, %edx +; X64-NEXT: adcl $0, %eax +; X64-NEXT: retq %sub = sub i32 %x, %y %shift = lshr i32 %z, 8 %mask = and i32 %shift, 1 @@ -172,32 +131,20 @@ define i32 @test_i32_sub_add_idx(i32 %x, i32 %y, i32 %z) { ; X86-LABEL: test_i32_sub_add_idx: ; X86: # %bb.0: -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: addl {{[0-9]+}}(%esp), %eax -; X86-NEXT: shrl %ecx -; X86-NEXT: andl $1, %ecx -; X86-NEXT: subl %ecx, %eax +; X86-NEXT: btl $1, {{[0-9]+}}(%esp) +; X86-NEXT: sbbl $0, %eax ; X86-NEXT: retl ; -; NOTBM-LABEL: test_i32_sub_add_idx: -; NOTBM: # %bb.0: -; NOTBM-NEXT: # kill: def $esi killed $esi def $rsi -; NOTBM-NEXT: # kill: def $edi killed $edi def $rdi -; NOTBM-NEXT: leal (%rdi,%rsi), %eax -; NOTBM-NEXT: shrl %edx -; NOTBM-NEXT: andl $1, %edx -; NOTBM-NEXT: subl %edx, %eax -; NOTBM-NEXT: retq -; -; TBM-LABEL: test_i32_sub_add_idx: -; TBM: # %bb.0: -; TBM-NEXT: # kill: def $esi killed $esi def $rsi -; TBM-NEXT: # kill: def $edi killed $edi def $rdi -; TBM-NEXT: leal (%rdi,%rsi), %eax -; TBM-NEXT: bextrl $257, %edx, %ecx # imm = 0x101 -; TBM-NEXT: subl %ecx, %eax -; TBM-NEXT: retq +; X64-LABEL: test_i32_sub_add_idx: +; X64: # %bb.0: +; X64-NEXT: # kill: def $esi killed $esi def $rsi +; X64-NEXT: # kill: def $edi killed $edi def $rdi +; X64-NEXT: leal (%rdi,%rsi), %eax +; X64-NEXT: btl $1, %edx +; X64-NEXT: sbbl $0, %eax +; X64-NEXT: retq %add = add i32 %y, %x %shift = lshr i32 %z, 1 %mask = and i32 %shift, 1 @@ -245,24 +192,18 @@ define i32 @test_i32_add_add_var(i32 %x, i32 %y, i32 %z, i32 %w) { ; X86-LABEL: test_i32_add_add_var: ; X86: # %bb.0: -; X86-NEXT: movb {{[0-9]+}}(%esp), %cl ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: addl {{[0-9]+}}(%esp), %edx -; X86-NEXT: shrl %cl, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %edx, %eax +; X86-NEXT: btl %ecx, %edx +; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax ; X86-NEXT: retl ; ; X64-LABEL: test_i32_add_add_var: ; X64: # %bb.0: -; X64-NEXT: # kill: def $esi killed $esi def $rsi -; X64-NEXT: # kill: def $edi killed $edi def $rdi -; X64-NEXT: leal (%rdi,%rsi), %eax -; X64-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-NEXT: shrl %cl, %edx -; X64-NEXT: andl $1, %edx -; X64-NEXT: addl %edx, %eax +; X64-NEXT: movl %edi, %eax +; X64-NEXT: btl %ecx, %edx +; X64-NEXT: adcl %esi, %eax ; X64-NEXT: retq %add = add i32 %y, %x %shift = lshr i32 %z, %w @@ -274,24 +215,18 @@ define i32 @test_i32_add_add_commute_var(i32 %x, i32 %y, i32 %z, i32 %w) { ; X86-LABEL: test_i32_add_add_commute_var: ; X86: # %bb.0: -; X86-NEXT: movb {{[0-9]+}}(%esp), %cl ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: addl {{[0-9]+}}(%esp), %edx -; X86-NEXT: shrl %cl, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %edx, %eax +; X86-NEXT: btl %ecx, %edx +; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax ; X86-NEXT: retl ; ; X64-LABEL: test_i32_add_add_commute_var: ; X64: # %bb.0: -; X64-NEXT: # kill: def $esi killed $esi def $rsi -; X64-NEXT: # kill: def $edi killed $edi def $rdi -; X64-NEXT: leal (%rdi,%rsi), %eax -; X64-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-NEXT: shrl %cl, %edx -; X64-NEXT: andl $1, %edx -; X64-NEXT: addl %edx, %eax +; X64-NEXT: movl %edi, %eax +; X64-NEXT: btl %ecx, %edx +; X64-NEXT: adcl %esi, %eax ; X64-NEXT: retq %add = add i32 %y, %x %shift = lshr i32 %z, %w @@ -303,24 +238,20 @@ define i32 @test_i32_add_sub_var(i32 %x, i32 %y, i32 %z, i32 %w) { ; X86-LABEL: test_i32_add_sub_var: ; X86: # %bb.0: -; X86-NEXT: movb {{[0-9]+}}(%esp), %cl -; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: subl {{[0-9]+}}(%esp), %edx -; X86-NEXT: shrl %cl, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %edx, %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: subl {{[0-9]+}}(%esp), %eax +; X86-NEXT: btl %ecx, %edx +; X86-NEXT: adcl $0, %eax ; X86-NEXT: retl ; ; X64-LABEL: test_i32_add_sub_var: ; X64: # %bb.0: -; X64-NEXT: # kill: def $edx killed $edx def $rdx -; X64-NEXT: # kill: def $edi killed $edi def $rdi -; X64-NEXT: subl %esi, %edi -; X64-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-NEXT: shrl %cl, %edx -; X64-NEXT: andl $1, %edx -; X64-NEXT: leal (%rdx,%rdi), %eax +; X64-NEXT: movl %edi, %eax +; X64-NEXT: subl %esi, %eax +; X64-NEXT: btl %ecx, %edx +; X64-NEXT: adcl $0, %eax ; X64-NEXT: retq %sub = sub i32 %x, %y %shift = lshr i32 %z, %w @@ -332,24 +263,20 @@ define i32 @test_i32_add_sub_commute_var(i32 %x, i32 %y, i32 %z, i32 %w) { ; X86-LABEL: test_i32_add_sub_commute_var: ; X86: # %bb.0: -; X86-NEXT: movb {{[0-9]+}}(%esp), %cl -; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: subl {{[0-9]+}}(%esp), %edx -; X86-NEXT: shrl %cl, %eax -; X86-NEXT: andl $1, %eax -; X86-NEXT: addl %edx, %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: subl {{[0-9]+}}(%esp), %eax +; X86-NEXT: btl %ecx, %edx +; X86-NEXT: adcl $0, %eax ; X86-NEXT: retl ; ; X64-LABEL: test_i32_add_sub_commute_var: ; X64: # %bb.0: -; X64-NEXT: # kill: def $edx killed $edx def $rdx -; X64-NEXT: # kill: def $edi killed $edi def $rdi -; X64-NEXT: subl %esi, %edi -; X64-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-NEXT: shrl %cl, %edx -; X64-NEXT: andl $1, %edx -; X64-NEXT: leal (%rdx,%rdi), %eax +; X64-NEXT: movl %edi, %eax +; X64-NEXT: subl %esi, %eax +; X64-NEXT: btl %ecx, %edx +; X64-NEXT: adcl $0, %eax ; X64-NEXT: retq %sub = sub i32 %x, %y %shift = lshr i32 %z, %w @@ -361,13 +288,12 @@ define i32 @test_i32_sub_add_var(i32 %x, i32 %y, i32 %z, i32 %w) { ; X86-LABEL: test_i32_sub_add_var: ; X86: # %bb.0: -; X86-NEXT: movb {{[0-9]+}}(%esp), %cl +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %edx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: addl {{[0-9]+}}(%esp), %eax -; X86-NEXT: shrl %cl, %edx -; X86-NEXT: andl $1, %edx -; X86-NEXT: subl %edx, %eax +; X86-NEXT: btl %ecx, %edx +; X86-NEXT: sbbl $0, %eax ; X86-NEXT: retl ; ; X64-LABEL: test_i32_sub_add_var: @@ -375,10 +301,8 @@ ; X64-NEXT: # kill: def $esi killed $esi def $rsi ; X64-NEXT: # kill: def $edi killed $edi def $rdi ; X64-NEXT: leal (%rdi,%rsi), %eax -; X64-NEXT: # kill: def $cl killed $cl killed $ecx -; X64-NEXT: shrl %cl, %edx -; X64-NEXT: andl $1, %edx -; X64-NEXT: subl %edx, %eax +; X64-NEXT: btl %ecx, %edx +; X64-NEXT: sbbl $0, %eax ; X64-NEXT: retq %add = add i32 %y, %x %shift = lshr i32 %z, %w