diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -31418,36 +31418,47 @@ AtomicRMWInst::BinOp Opc = AI->getOperation(); if (Opc == AtomicRMWInst::Add) { if (match(I, m_c_ICmp(Pred, m_Sub(m_ZeroInt(), m_Specific(Op)), m_Value()))) - return Pred == CmpInst::ICMP_EQ; - if (match(I, m_OneUse(m_c_Add(m_Specific(Op), m_Value()))) && - match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) - return Pred == CmpInst::ICMP_SLT; + return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE; + if (match(I, m_OneUse(m_c_Add(m_Specific(Op), m_Value())))) { + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) + return Pred == CmpInst::ICMP_SLT; + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) + return Pred == CmpInst::ICMP_SGT; + } return false; } if (Opc == AtomicRMWInst::Sub) { if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value()))) - return Pred == CmpInst::ICMP_EQ; - if (match(I, m_OneUse(m_Sub(m_Value(), m_Specific(Op)))) && - match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) - return Pred == CmpInst::ICMP_SLT; + return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE; + if (match(I, m_OneUse(m_Sub(m_Value(), m_Specific(Op))))) { + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) + return Pred == CmpInst::ICMP_SLT; + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) + return Pred == CmpInst::ICMP_SGT; + } return false; } - if (Opc == AtomicRMWInst::Or) { - if (match(I, m_OneUse(m_c_Or(m_Specific(Op), m_Value()))) && - match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) - return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLT; - } - if (Opc == AtomicRMWInst::And) { - if (match(I, m_OneUse(m_c_And(m_Specific(Op), m_Value()))) && - match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) - return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLT; + if ((Opc == AtomicRMWInst::Or && + match(I, m_OneUse(m_c_Or(m_Specific(Op), m_Value())))) || + (Opc == AtomicRMWInst::And && + match(I, m_OneUse(m_c_And(m_Specific(Op), m_Value()))))) { + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) + return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE || + Pred == CmpInst::ICMP_SLT; + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) + return Pred == CmpInst::ICMP_SGT; + return false; } if (Opc == AtomicRMWInst::Xor) { if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value()))) - return Pred == CmpInst::ICMP_EQ; - if (match(I, m_OneUse(m_c_Xor(m_Specific(Op), m_Value()))) && - match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) - return Pred == CmpInst::ICMP_SLT; + return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE; + if (match(I, m_OneUse(m_c_Xor(m_Specific(Op), m_Value())))) { + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) + return Pred == CmpInst::ICMP_SLT; + if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) + return Pred == CmpInst::ICMP_SGT; + } + return false; } return false; @@ -31464,10 +31475,24 @@ assert(TempI->hasOneUse() && "Must have one use"); ICI = cast(TempI->user_back()); } + X86::CondCode CC = X86::COND_INVALID; ICmpInst::Predicate Pred = ICI->getPredicate(); - assert((Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_SLT) && - "Not supported Pred"); - X86::CondCode CC = Pred == CmpInst::ICMP_EQ ? X86::COND_E : X86::COND_S; + switch (Pred) { + default: + llvm_unreachable("Not supported Pred"); + case CmpInst::ICMP_EQ: + CC = X86::COND_E; + break; + case CmpInst::ICMP_NE: + CC = X86::COND_NE; + break; + case CmpInst::ICMP_SLT: + CC = X86::COND_S; + break; + case CmpInst::ICMP_SGT: + CC = X86::COND_NS; + break; + } Intrinsic::ID IID = Intrinsic::not_intrinsic; switch (AI->getOperation()) { default: diff --git a/llvm/test/CodeGen/X86/pr58685.ll b/llvm/test/CodeGen/X86/pr58685.ll --- a/llvm/test/CodeGen/X86/pr58685.ll +++ b/llvm/test/CodeGen/X86/pr58685.ll @@ -122,9 +122,7 @@ define i1 @lock_add_setne(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_add_setne: ; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %eax -; CHECK-NEXT: lock xaddl %eax, (%rdi) -; CHECK-NEXT: addl %esi, %eax +; CHECK-NEXT: lock addl %esi, (%rdi) ; CHECK-NEXT: setne %al ; CHECK-NEXT: retq %3 = atomicrmw add ptr %0, i32 %1 seq_cst, align 4 @@ -136,9 +134,7 @@ define i1 @lock_add_setns(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_add_setns: ; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %eax -; CHECK-NEXT: lock xaddl %eax, (%rdi) -; CHECK-NEXT: addl %esi, %eax +; CHECK-NEXT: lock addl %esi, (%rdi) ; CHECK-NEXT: setns %al ; CHECK-NEXT: retq %3 = atomicrmw add ptr %0, i32 %1 seq_cst, align 4 @@ -150,10 +146,7 @@ define i1 @lock_sub_setne(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_sub_setne: ; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %eax -; CHECK-NEXT: negl %eax -; CHECK-NEXT: lock xaddl %eax, (%rdi) -; CHECK-NEXT: cmpl %esi, %eax +; CHECK-NEXT: lock subl %esi, (%rdi) ; CHECK-NEXT: setne %al ; CHECK-NEXT: retq %3 = atomicrmw sub ptr %0, i32 %1 seq_cst, align 4 @@ -164,10 +157,7 @@ define i1 @lock_sub_setns(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_sub_setns: ; CHECK: # %bb.0: -; CHECK-NEXT: movl %esi, %eax -; CHECK-NEXT: negl %eax -; CHECK-NEXT: lock xaddl %eax, (%rdi) -; CHECK-NEXT: cmpl %esi, %eax +; CHECK-NEXT: lock subl %esi, (%rdi) ; CHECK-NEXT: setns %al ; CHECK-NEXT: retq %3 = atomicrmw sub ptr %0, i32 %1 seq_cst, align 4 @@ -179,16 +169,7 @@ define i1 @lock_or_setne(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_or_setne: ; CHECK: # %bb.0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB14_1: # %atomicrmw.start -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: movl %eax, %ecx -; CHECK-NEXT: orl %esi, %ecx -; CHECK-NEXT: lock cmpxchgl %ecx, (%rdi) -; CHECK-NEXT: jne .LBB14_1 -; CHECK-NEXT: # %bb.2: # %atomicrmw.end -; CHECK-NEXT: orl %esi, %eax +; CHECK-NEXT: lock orl %esi, (%rdi) ; CHECK-NEXT: setne %al ; CHECK-NEXT: retq %3 = atomicrmw or ptr %0, i32 %1 seq_cst, align 4 @@ -200,16 +181,7 @@ define i1 @lock_or_setns(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_or_setns: ; CHECK: # %bb.0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB15_1: # %atomicrmw.start -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: movl %eax, %ecx -; CHECK-NEXT: orl %esi, %ecx -; CHECK-NEXT: lock cmpxchgl %ecx, (%rdi) -; CHECK-NEXT: jne .LBB15_1 -; CHECK-NEXT: # %bb.2: # %atomicrmw.end -; CHECK-NEXT: orl %esi, %eax +; CHECK-NEXT: lock orl %esi, (%rdi) ; CHECK-NEXT: setns %al ; CHECK-NEXT: retq %3 = atomicrmw or ptr %0, i32 %1 seq_cst, align 4 @@ -221,16 +193,7 @@ define i1 @lock_and_setne(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_and_setne: ; CHECK: # %bb.0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB16_1: # %atomicrmw.start -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: movl %eax, %ecx -; CHECK-NEXT: andl %esi, %ecx -; CHECK-NEXT: lock cmpxchgl %ecx, (%rdi) -; CHECK-NEXT: jne .LBB16_1 -; CHECK-NEXT: # %bb.2: # %atomicrmw.end -; CHECK-NEXT: testl %esi, %eax +; CHECK-NEXT: lock andl %esi, (%rdi) ; CHECK-NEXT: setne %al ; CHECK-NEXT: retq %3 = atomicrmw and ptr %0, i32 %1 seq_cst, align 4 @@ -242,16 +205,7 @@ define i1 @lock_and_setns(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_and_setns: ; CHECK: # %bb.0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB17_1: # %atomicrmw.start -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: movl %eax, %ecx -; CHECK-NEXT: andl %esi, %ecx -; CHECK-NEXT: lock cmpxchgl %ecx, (%rdi) -; CHECK-NEXT: jne .LBB17_1 -; CHECK-NEXT: # %bb.2: # %atomicrmw.end -; CHECK-NEXT: testl %esi, %eax +; CHECK-NEXT: lock andl %esi, (%rdi) ; CHECK-NEXT: setns %al ; CHECK-NEXT: retq %3 = atomicrmw and ptr %0, i32 %1 seq_cst, align 4 @@ -263,16 +217,7 @@ define i1 @lock_xor_setne(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_xor_setne: ; CHECK: # %bb.0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB18_1: # %atomicrmw.start -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: movl %eax, %ecx -; CHECK-NEXT: xorl %esi, %ecx -; CHECK-NEXT: lock cmpxchgl %ecx, (%rdi) -; CHECK-NEXT: jne .LBB18_1 -; CHECK-NEXT: # %bb.2: # %atomicrmw.end -; CHECK-NEXT: cmpl %esi, %eax +; CHECK-NEXT: lock xorl %esi, (%rdi) ; CHECK-NEXT: setne %al ; CHECK-NEXT: retq %3 = atomicrmw xor ptr %0, i32 %1 seq_cst, align 4 @@ -283,16 +228,7 @@ define i1 @lock_xor_setns(ptr %0, i32 %1) nounwind { ; CHECK-LABEL: lock_xor_setns: ; CHECK: # %bb.0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB19_1: # %atomicrmw.start -; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: movl %eax, %ecx -; CHECK-NEXT: xorl %esi, %ecx -; CHECK-NEXT: lock cmpxchgl %ecx, (%rdi) -; CHECK-NEXT: jne .LBB19_1 -; CHECK-NEXT: # %bb.2: # %atomicrmw.end -; CHECK-NEXT: xorl %esi, %eax +; CHECK-NEXT: lock xorl %esi, (%rdi) ; CHECK-NEXT: setns %al ; CHECK-NEXT: retq %3 = atomicrmw xor ptr %0, i32 %1 seq_cst, align 4