Index: include/llvm/Target/TargetLowering.h =================================================================== --- include/llvm/Target/TargetLowering.h +++ include/llvm/Target/TargetLowering.h @@ -1097,6 +1097,14 @@ } /// @} + // Emits code that executes when the comparison result in the ll/sc + // expansion of a cmpxchg instruction is such that the store-conditional will + // not execute. This makes it possible to balance out the load-linked with + // a dedicated instruction, if desired. + // E.g., on ARM, if ldrex isn't followed by strex, the exclusive monitor would + // be unnecessarily held, except if clrex, inserted by this hook, is executed. + virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const {} + /// Returns true if the given (atomic) store should be expanded by the /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input. virtual bool shouldExpandAtomicStoreInIR(StoreInst *SI) const { Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -374,7 +374,7 @@ // %loaded = @load.linked(%addr) // %should_store = icmp eq %loaded, %desired // br i1 %should_store, label %cmpxchg.trystore, - // label %cmpxchg.failure + // label %cmpxchg.nostore // cmpxchg.trystore: // %stored = @store_conditional(%new, %addr) // %success = icmp eq i32 %stored, 0 @@ -382,6 +382,9 @@ // cmpxchg.success: // fence? // br label %cmpxchg.end + // cmpxchg.nostore: + // @load_linked_fail_balance()? + // br label %cmpxchg.failure // cmpxchg.failure: // fence? // br label %cmpxchg.end @@ -392,7 +395,8 @@ // [...] BasicBlock *ExitBB = BB->splitBasicBlock(CI, "cmpxchg.end"); auto FailureBB = BasicBlock::Create(Ctx, "cmpxchg.failure", F, ExitBB); - auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, FailureBB); + auto NoStoreBB = BasicBlock::Create(Ctx, "cmpxchg.nostore", F, FailureBB); + auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, NoStoreBB); auto TryStoreBB = BasicBlock::Create(Ctx, "cmpxchg.trystore", F, SuccessBB); auto LoopBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, TryStoreBB); @@ -416,7 +420,7 @@ // If the cmpxchg doesn't actually need any ordering when it fails, we can // jump straight past that fence instruction (if it exists). - Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB); + Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB); Builder.SetInsertPoint(TryStoreBB); Value *StoreSuccess = TLI->emitStoreConditional( @@ -432,6 +436,13 @@ /*IsLoad=*/true); Builder.CreateBr(ExitBB); + Builder.SetInsertPoint(NoStoreBB); + // In the failing case, where we don't execute the store-conditional, the + // target might want to balance out the load-linked with a dedicated + // instruction (e.g., on ARM, clearing the exclusive monitor). + TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder); + Builder.CreateBr(FailureBB); + Builder.SetInsertPoint(FailureBB); TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true, /*IsLoad=*/true); Index: lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.h +++ lib/Target/AArch64/AArch64ISelLowering.h @@ -348,6 +348,8 @@ Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override; + void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const override; + TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9682,6 +9682,13 @@ cast(Addr->getType())->getElementType()); } +void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance( + IRBuilder<> &Builder) const { + Module *M = Builder.GetInsertBlock()->getParent()->getParent(); + Builder.CreateCall( + llvm::Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex)); +} + Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const { Index: lib/Target/ARM/ARMISelLowering.h =================================================================== --- lib/Target/ARM/ARMISelLowering.h +++ lib/Target/ARM/ARMISelLowering.h @@ -421,6 +421,8 @@ Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override; + void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const override; + Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const override; Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -11622,6 +11622,12 @@ cast(Addr->getType())->getElementType()); } +void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance( + IRBuilder<> &Builder) const { + Module *M = Builder.GetInsertBlock()->getParent()->getParent(); + Builder.CreateCall(llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_clrex)); +} + Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const { Index: test/CodeGen/AArch64/arm64-atomic.ll =================================================================== --- test/CodeGen/AArch64/arm64-atomic.ll +++ test/CodeGen/AArch64/arm64-atomic.ll @@ -2,13 +2,17 @@ define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 { ; CHECK-LABEL: val_compare_and_swap: -; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x0] +; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 +; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: +; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]] ; CHECK-NEXT: cmp [[RESULT]], w1 -; CHECK-NEXT: b.ne [[LABEL2:.?LBB[0-9]+_[0-9]+]] -; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], w2, [x0] -; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[LABEL]] -; CHECK-NEXT: [[LABEL2]]: +; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] +; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]] +; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] +; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] +; CHECK-NEXT: [[FAILBB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: [[EXITBB]]: %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val @@ -17,13 +21,16 @@ define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 { ; CHECK-LABEL: val_compare_and_swap_from_load: ; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2] -; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]: +; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: ; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x0] ; CHECK-NEXT: cmp [[RESULT]], w1 -; CHECK-NEXT: b.ne [[LABEL2:.?LBB[0-9]+_[0-9]+]] +; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], [[NEW]], [x0] -; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[LABEL]] -; CHECK-NEXT: [[LABEL2]]: +; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] +; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] +; CHECK-NEXT: [[FAILBB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: [[EXITBB]]: %new = load i32, i32* %pnew %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acquire acquire %val = extractvalue { i32, i1 } %pair, 0 @@ -32,13 +39,17 @@ define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 { ; CHECK-LABEL: val_compare_and_swap_rel: -; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x0] +; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 +; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: +; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]] ; CHECK-NEXT: cmp [[RESULT]], w1 -; CHECK-NEXT: b.ne [[LABEL2:.?LBB[0-9]+_[0-9]+]] -; CHECK-NEXT: stlxr [[SCRATCH_REG:w[0-9]+]], w2, [x0] -; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[LABEL]] -; CHECK-NEXT: [[LABEL2]]: +; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] +; CHECK-NEXT: stlxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]] +; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] +; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] +; CHECK-NEXT: [[FAILBB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: [[EXITBB]]: %pair = cmpxchg i32* %p, i32 %cmp, i32 %new acq_rel monotonic %val = extractvalue { i32, i1 } %pair, 0 ret i32 %val @@ -47,13 +58,16 @@ define i64 @val_compare_and_swap_64(i64* %p, i64 %cmp, i64 %new) #0 { ; CHECK-LABEL: val_compare_and_swap_64: ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 -; CHECK-NEXT: [[LABEL:.?LBB[0-9]+_[0-9]+]]: +; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: ; CHECK-NEXT: ldxr [[RESULT:x[0-9]+]], [x[[ADDR]]] ; CHECK-NEXT: cmp [[RESULT]], x1 -; CHECK-NEXT: b.ne [[LABEL2:.?LBB[0-9]+_[0-9]+]] +; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] ; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], x2, [x[[ADDR]]] -; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[LABEL]] -; CHECK-NEXT: [[LABEL2]]: +; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] +; CHECK-NEXT: b [[EXITBB:.?LBB[0-9_]+]] +; CHECK-NEXT: [[FAILBB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: [[EXITBB]]: %pair = cmpxchg i64* %p, i64 %cmp, i64 %new monotonic monotonic %val = extractvalue { i64, i1 } %pair, 0 ret i64 %val @@ -61,13 +75,13 @@ define i32 @fetch_and_nand(i32* %p) #0 { ; CHECK-LABEL: fetch_and_nand: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: +; CHECK: [[TRYBB:.?LBB[0-9_]+]]: ; CHECK: ldxr w[[DEST_REG:[0-9]+]], [x0] ; CHECK: mvn [[TMP_REG:w[0-9]+]], w[[DEST_REG]] ; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], [[TMP_REG]], #0xfffffff8 ; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] -; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] +; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] ; CHECK: mov x0, x[[DEST_REG]] %val = atomicrmw nand i32* %p, i32 7 release ret i32 %val @@ -76,12 +90,12 @@ define i64 @fetch_and_nand_64(i64* %p) #0 { ; CHECK-LABEL: fetch_and_nand_64: ; CHECK: mov x[[ADDR:[0-9]+]], x0 -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: +; CHECK: [[TRYBB:.?LBB[0-9_]+]]: ; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR]]] ; CHECK: mvn w[[TMP_REG:[0-9]+]], w[[DEST_REG]] ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], x[[TMP_REG]], #0xfffffffffffffff8 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] -; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] +; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] %val = atomicrmw nand i64* %p, i64 7 acq_rel ret i64 %val @@ -90,12 +104,12 @@ define i32 @fetch_and_or(i32* %p) #0 { ; CHECK-LABEL: fetch_and_or: ; CHECK: movz [[OLDVAL_REG:w[0-9]+]], #0x5 -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: +; CHECK: [[TRYBB:.?LBB[0-9_]+]]: ; CHECK: ldaxr w[[DEST_REG:[0-9]+]], [x0] ; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], w[[DEST_REG]], [[OLDVAL_REG]] ; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] -; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] +; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] ; CHECK: mov x0, x[[DEST_REG]] %val = atomicrmw or i32* %p, i32 5 seq_cst ret i32 %val @@ -104,11 +118,11 @@ define i64 @fetch_and_or_64(i64* %p) #0 { ; CHECK: fetch_and_or_64: ; CHECK: mov x[[ADDR:[0-9]+]], x0 -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: +; CHECK: [[TRYBB:.?LBB[0-9_]+]]: ; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR]]] ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], [[DEST_REG]], #0x7 ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] -; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] +; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] %val = atomicrmw or i64* %p, i64 7 monotonic ret i64 %val } Index: test/CodeGen/AArch64/atomic-ops.ll =================================================================== --- test/CodeGen/AArch64/atomic-ops.ll +++ test/CodeGen/AArch64/atomic-ops.ll @@ -893,6 +893,8 @@ ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] ; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]] ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] +; CHECK: [[GET_OUT]]: +; CHECK: clrex ; CHECK-NOT: dmb ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] @@ -916,6 +918,8 @@ ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] ; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]] ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] +; CHECK: [[GET_OUT]]: +; CHECK: clrex ; CHECK-NOT: dmb ; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] @@ -927,21 +931,21 @@ %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic %old = extractvalue { i32, i1 } %pair, 0 +; CHECK: mov {{[xw]}}[[WANTED:[0-9]+]], {{[xw]}}0 + ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 ; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]: ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]] - ; w0 below is a reasonable guess but could change: it certainly comes into the - ; function there. -; CHECK-NEXT: cmp w[[OLD]], w0 +; CHECK-NEXT: cmp w[[OLD]], w[[WANTED]] ; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]] ; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]] ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] +; CHECK: [[GET_OUT]]: +; CHECK: clrex ; CHECK-NOT: dmb - -; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]] ret i32 %old } @@ -963,6 +967,8 @@ ; As above, w1 is a reasonable guess. ; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]] ; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]] +; CHECK: [[GET_OUT]]: +; CHECK: clrex ; CHECK-NOT: dmb ; CHECK: str x[[OLD]], Index: test/CodeGen/ARM/atomic-cmp.ll =================================================================== --- test/CodeGen/ARM/atomic-cmp.ll +++ test/CodeGen/ARM/atomic-cmp.ll @@ -6,10 +6,12 @@ ; ARM-LABEL: t: ; ARM: ldrexb ; ARM: strexb +; ARM: clrex ; T2-LABEL: t: -; T2: ldrexb ; T2: strexb +; T2: ldrexb +; T2: clrex %tmp0 = cmpxchg i8* %a, i8 %b, i8 %c monotonic monotonic %tmp1 = extractvalue { i8, i1 } %tmp0, 0 ret i8 %tmp1 Index: test/CodeGen/ARM/atomic-cmpxchg.ll =================================================================== --- test/CodeGen/ARM/atomic-cmpxchg.ll +++ test/CodeGen/ARM/atomic-cmpxchg.ll @@ -1,8 +1,8 @@ -; RUN: llc < %s -mtriple=arm-linux-gnueabi -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-ARM -; RUN: llc < %s -mtriple=thumb-linux-gnueabi -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-THUMB +; RUN: llc < %s -mtriple=arm-linux-gnueabi -asm-verbose=false -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-ARM +; RUN: llc < %s -mtriple=thumb-linux-gnueabi -asm-verbose=false -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-THUMB -; RUN: llc < %s -mtriple=armv7-linux-gnueabi -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-ARMV7 -; RUN: llc < %s -mtriple=thumbv7-linux-gnueabi -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-THUMBV7 +; RUN: llc < %s -mtriple=armv7-linux-gnueabi -asm-verbose=false -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-ARMV7 +; RUN: llc < %s -mtriple=thumbv7-linux-gnueabi -asm-verbose=false -verify-machineinstrs | FileCheck %s -check-prefix=CHECK-THUMBV7 define zeroext i1 @test_cmpxchg_res_i8(i8* %addr, i8 %desired, i8 zeroext %new) { entry: @@ -30,24 +30,39 @@ ; CHECK-THUMB: push {[[R2]]} ; CHECK-THUMB: pop {r0} -; CHECK-ARMV7-LABEL: test_cmpxchg_res_i8 -; CHECK-ARMV7: ldrexb [[R3:r[0-9]+]], [r0] -; CHECK-ARMV7: mov [[R1:r[0-9]+]], #0 -; CHECK-ARMV7: cmp [[R3]], {{r[0-9]+}} -; CHECK-ARMV7: bne -; CHECK-ARMV7: strexb [[R3]], {{r[0-9]+}}, [{{r[0-9]+}}] -; CHECK-ARMV7: mov [[R1]], #1 -; CHECK-ARMV7: cmp [[R3]], #0 -; CHECK-ARMV7: bne -; CHECK-ARMV7: mov r0, [[R1]] +; CHECK-ARMV7-LABEL: test_cmpxchg_res_i8: +; CHECK-ARMV7-NEXT: .fnstart +; CHECK-ARMV7-NEXT: uxtb [[DESIRED:r[0-9]+]], r1 +; CHECK-ARMV7-NEXT: [[TRY:.LBB[0-9_]+]]: +; CHECK-ARMV7-NEXT: ldrexb [[LD:r[0-9]+]], [r0] +; CHECK-ARMV7-NEXT: cmp [[LD]], [[DESIRED]] +; CHECK-ARMV7-NEXT: bne [[FAIL:.LBB[0-9_]+]] +; CHECK-ARMV7-NEXT: strexb [[SUCCESS:r[0-9]+]], r2, [r0] +; CHECK-ARMV7-NEXT: mov [[RES:r[0-9]+]], #1 +; CHECK-ARMV7-NEXT: cmp [[SUCCESS]], #0 +; CHECK-ARMV7-NEXT: bne [[TRY]] +; CHECK-ARMV7-NEXT: b [[END:.LBB[0-9_]+]] +; CHECK-ARMV7-NEXT: [[FAIL]]: +; CHECK-ARMV7-NEXT: clrex +; CHECK-ARMV7-NEXT: mov [[RES]], #0 +; CHECK-ARMV7-NEXT: [[END]]: +; CHECK-ARMV7-NEXT: mov r0, [[RES]] +; CHECK-ARMV7-NEXT: bx lr -; CHECK-THUMBV7-LABEL: test_cmpxchg_res_i8 -; CHECK-THUMBV7: ldrexb [[R3:r[0-9]+]], [r0] -; CHECK-THUMBV7: cmp [[R3]], {{r[0-9]+}} -; CHECK-THUMBV7: movne r0, #0 -; CHECK-THUMBV7: bxne lr -; CHECK-THUMBV7: strexb [[R3]], {{r[0-9]+}}, [{{r[0-9]+}}] -; CHECK-THUMBV7: cmp [[R3]], #0 -; CHECK-THUMBV7: itt eq -; CHECK-THUMBV7: moveq r0, #1 -; CHECK-THUMBV7: bxeq lr +; CHECK-THUMBV7-LABEL: test_cmpxchg_res_i8: +; CHECK-THUMBV7-NEXT: .fnstart +; CHECK-THUMBV7-NEXT: uxtb [[DESIRED:r[0-9]+]], r1 +; CHECK-THUMBV7-NEXT: b [[TRYLD:.LBB[0-9_]+]] +; CHECK-THUMBV7-NEXT: [[TRYST:.LBB[0-9_]+]]: +; CHECK-THUMBV7-NEXT: strexb [[SUCCESS:r[0-9]+]], r2, [r0] +; CHECK-THUMBV7-NEXT: cmp [[SUCCESS]], #0 +; CHECK-THUMBV7-NEXT: itt eq +; CHECK-THUMBV7-NEXT: moveq r0, #1 +; CHECK-THUMBV7-NEXT: bxeq lr +; CHECK-THUMBV7-NEXT: [[TRYLD]]: +; CHECK-THUMBV7-NEXT: ldrexb [[LD:r[0-9]+]], [r0] +; CHECK-THUMBV7-NEXT: cmp [[LD]], [[DESIRED]] +; CHECK-THUMBV7-NEXT: beq [[TRYST:.LBB[0-9_]+]] +; CHECK-THUMBV7-NEXT: clrex +; CHECK-THUMBV7-NEXT: movs r0, #0 +; CHECK-THUMBV7-NEXT: bx lr Index: test/CodeGen/ARM/atomic-op.ll =================================================================== --- test/CodeGen/ARM/atomic-op.ll +++ test/CodeGen/ARM/atomic-op.ll @@ -1,5 +1,5 @@ -; RUN: llc < %s -mtriple=armv7-apple-ios -verify-machineinstrs | FileCheck %s -; RUN: llc < %s -mtriple=thumbv7-apple-ios -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -mtriple=armv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix CHECK-ARMV7 +; RUN: llc < %s -mtriple=thumbv7-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-T2 ; RUN: llc < %s -mtriple=thumbv6-apple-ios -verify-machineinstrs | FileCheck %s --check-prefix=CHECK-T1 ; RUN: llc < %s -mtriple=thumbv6-apple-ios -verify-machineinstrs -mcpu=cortex-m0 | FileCheck %s --check-prefix=CHECK-M0 ; RUN: llc < %s -mtriple=thumbv7--none-eabi -thread-model single -verify-machineinstrs | FileCheck %s --check-prefix=CHECK-BAREMETAL @@ -272,16 +272,31 @@ %pair = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic %oldval = extractvalue { i32, i1 } %pair, 0 -; CHECK: dmb ish -; CHECK: [[LOOP_BB:\.?LBB[0-9]+_1]]: -; CHECK: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]] -; CHECK: cmp [[OLDVAL]], r1 -; CHECK: bxne lr -; CHECK: strex [[SUCCESS:r[0-9]+]], r2, [r[[ADDR]]] -; CHECK: cmp [[SUCCESS]], #0 -; CHECK: bne [[LOOP_BB]] -; CHECK: dmb ish -; CHECK: bx lr +; CHECK-ARMV7: dmb ish +; CHECK-ARMV7: [[LOOP_BB:\.?LBB[0-9]+_1]]: +; CHECK-ARMV7: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]] +; CHECK-ARMV7: cmp [[OLDVAL]], r1 +; CHECK-ARMV7: bne [[FAIL_BB:\.?LBB[0-9]+_[0-9]+]] +; CHECK-ARMV7: strex [[SUCCESS:r[0-9]+]], r2, [r[[ADDR]]] +; CHECK-ARMV7: cmp [[SUCCESS]], #0 +; CHECK-ARMV7: bne [[LOOP_BB]] +; CHECK-ARMV7: dmb ish +; CHECK-ARMV7: bx lr +; CHECK-ARMV7: [[FAIL_BB]]: +; CHECK-ARMV7: clrex +; CHECK-ARMV7: bx lr + +; CHECK-T2: dmb ish +; CHECK-T2: [[LOOP_BB:\.?LBB[0-9]+_1]]: +; CHECK-T2: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]] +; CHECK-T2: cmp [[OLDVAL]], r1 +; CHECK-T2: clrexne +; CHECK-T2: bxne lr +; CHECK-T2: strex [[SUCCESS:r[0-9]+]], r2, [r[[ADDR]]] +; CHECK-T2: cmp [[SUCCESS]], #0 +; CHECK-T2: dmbeq ish +; CHECK-T2: bxeq lr +; CHECK-T2: b [[LOOP_BB]] ret i32 %oldval } @@ -295,11 +310,14 @@ ; CHECK: [[LOOP_BB:\.?LBB[0-9]+_1]]: ; CHECK: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]] ; CHECK: cmp [[OLDVAL]], r1 -; CHECK: bne [[END_BB:\.?LBB[0-9]+_[0-9]+]] +; CHECK: bne [[FAIL_BB:\.?LBB[0-9]+_[0-9]+]] ; CHECK: strex [[SUCCESS:r[0-9]+]], r2, [r[[ADDR]]] ; CHECK: cmp [[SUCCESS]], #0 ; CHECK: bne [[LOOP_BB]] -; CHECK: [[END_BB]]: +; CHECK: b [[END_BB:\.?LBB[0-9]+_[0-9]+]] +; CHECK: [[FAIL_BB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: [[END_BB]]: ; CHECK: dmb ish ; CHECK: bx lr Index: test/CodeGen/ARM/atomic-ops-v8.ll =================================================================== --- test/CodeGen/ARM/atomic-ops-v8.ll +++ test/CodeGen/ARM/atomic-ops-v8.ll @@ -1055,24 +1055,30 @@ %old = extractvalue { i8, i1 } %pair, 0 ; CHECK-NOT: dmb ; CHECK-NOT: mcr -; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8 -; CHECK: movt r[[ADDR]], :upper16:var8 +; CHECK-DAG: movw r[[ADDR:[0-9]+]], :lower16:var8 +; CHECK-DAG: movt r[[ADDR]], :upper16:var8 +; CHECK-THUMB-DAG: mov r[[WANTED:[0-9]+]], r0 ; CHECK: .LBB{{[0-9]+}}_1: ; CHECK: ldaexb r[[OLD:[0-9]+]], [r[[ADDR]]] ; r0 below is a reasonable guess but could change: it certainly comes into the ; function there. -; CHECK-NEXT: cmp r[[OLD]], r0 +; CHECK-ARM-NEXT: cmp r[[OLD]], r0 +; CHECK-THUMB-NEXT: cmp r[[OLD]], r[[WANTED]] ; CHECK-NEXT: bne .LBB{{[0-9]+}}_3 ; CHECK-NEXT: BB#2: ; As above, r1 is a reasonable guess. -; CHECK: strexb [[STATUS:r[0-9]+]], r1, {{.*}}[[ADDR]] +; CHECK: strexb [[STATUS:r[0-9]+]], r1, [r[[ADDR]]] ; CHECK-NEXT: cmp [[STATUS]], #0 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1 +; CHECK-NEXT: b .LBB{{[0-9]+}}_4 +; CHECK-NEXT: .LBB{{[0-9]+}}_3: +; CHECK-NEXT: clrex +; CHECK-NEXT: .LBB{{[0-9]+}}_4: ; CHECK-NOT: dmb ; CHECK-NOT: mcr -; CHECK: mov r0, r[[OLD]] +; CHECK-ARM: mov r0, r[[OLD]] ret i8 %old } @@ -1082,24 +1088,30 @@ %old = extractvalue { i16, i1 } %pair, 0 ; CHECK-NOT: dmb ; CHECK-NOT: mcr -; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16 -; CHECK: movt r[[ADDR]], :upper16:var16 +; CHECK-DAG: movw r[[ADDR:[0-9]+]], :lower16:var16 +; CHECK-DAG: movt r[[ADDR]], :upper16:var16 +; CHECK-THUMB-DAG: mov r[[WANTED:[0-9]+]], r0 ; CHECK: .LBB{{[0-9]+}}_1: ; CHECK: ldaexh r[[OLD:[0-9]+]], [r[[ADDR]]] ; r0 below is a reasonable guess but could change: it certainly comes into the ; function there. -; CHECK-NEXT: cmp r[[OLD]], r0 +; CHECK-ARM-NEXT: cmp r[[OLD]], r0 +; CHECK-THUMB-NEXT: cmp r[[OLD]], r[[WANTED]] ; CHECK-NEXT: bne .LBB{{[0-9]+}}_3 ; CHECK-NEXT: BB#2: ; As above, r1 is a reasonable guess. ; CHECK: stlexh [[STATUS:r[0-9]+]], r1, [r[[ADDR]]] ; CHECK-NEXT: cmp [[STATUS]], #0 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1 +; CHECK-NEXT: b .LBB{{[0-9]+}}_4 +; CHECK-NEXT: .LBB{{[0-9]+}}_3: +; CHECK-NEXT: clrex +; CHECK-NEXT: .LBB{{[0-9]+}}_4: ; CHECK-NOT: dmb ; CHECK-NOT: mcr -; CHECK: mov r0, r[[OLD]] +; CHECK-ARM: mov r0, r[[OLD]] ret i16 %old } @@ -1124,6 +1136,10 @@ ; CHECK: stlex [[STATUS:r[0-9]+]], r1, [r[[ADDR]]] ; CHECK-NEXT: cmp [[STATUS]], #0 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1 +; CHECK-NEXT: b .LBB{{[0-9]+}}_4 +; CHECK-NEXT: .LBB{{[0-9]+}}_3: +; CHECK-NEXT: clrex +; CHECK-NEXT: .LBB{{[0-9]+}}_4: ; CHECK-NOT: dmb ; CHECK-NOT: mcr @@ -1158,6 +1174,10 @@ ; CHECK: strexd [[STATUS:r[0-9]+]], r2, r3, [r[[ADDR]]] ; CHECK-NEXT: cmp [[STATUS]], #0 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1 +; CHECK-NEXT: b .LBB{{[0-9]+}}_4 +; CHECK-NEXT: .LBB{{[0-9]+}}_3: +; CHECK-NEXT: clrex +; CHECK-NEXT: .LBB{{[0-9]+}}_4: ; CHECK-NOT: dmb ; CHECK-NOT: mcr Index: test/CodeGen/ARM/cmpxchg-weak.ll =================================================================== --- test/CodeGen/ARM/cmpxchg-weak.ll +++ test/CodeGen/ARM/cmpxchg-weak.ll @@ -5,16 +5,24 @@ %pair = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst monotonic %oldval = extractvalue { i32, i1 } %pair, 0 -; CHECK: dmb ish -; CHECK: ldrex [[LOADED:r[0-9]+]], [r0] -; CHECK: cmp [[LOADED]], r1 -; CHECK: strexeq [[SUCCESS:r[0-9]+]], r2, [r0] -; CHECK: cmpeq [[SUCCESS]], #0 -; CHECK: bne [[DONE:LBB[0-9]+_[0-9]+]] -; CHECK: dmb ish -; CHECK: [[DONE]]: -; CHECK: str r3, [r0] -; CHECK: bx lr +; CHECK-NEXT: BB#0: +; CHECK-NEXT: dmb ish +; CHECK-NEXT: ldrex [[LOADED:r[0-9]+]], [r0] +; CHECK-NEXT: cmp [[LOADED]], r1 +; CHECK-NEXT: bne [[LDFAILBB:LBB[0-9]+_[0-9]+]] +; CHECK-NEXT: BB#1: +; CHECK-NEXT: strex [[SUCCESS:r[0-9]+]], r2, [r0] +; CHECK-NEXT: cmp [[SUCCESS]], #0 +; CHECK-NEXT: bne [[FAILBB:LBB[0-9]+_[0-9]+]] +; CHECK-NEXT: BB#2: +; CHECK-NEXT: dmb ish +; CHECK-NEXT: str r3, [r0] +; CHECK-NEXT: bx lr +; CHECK-NEXT: [[LDFAILBB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: [[FAILBB]]: +; CHECK-NEXT: str r3, [r0] +; CHECK-NEXT: bx lr store i32 %oldval, i32* %addr ret void @@ -27,17 +35,23 @@ %pair = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst monotonic %success = extractvalue { i32, i1 } %pair, 1 -; CHECK: dmb ish -; CHECK: mov r0, #0 -; CHECK: ldrex [[LOADED:r[0-9]+]], [r1] -; CHECK: cmp [[LOADED]], r2 -; CHECK: strexeq [[STATUS:r[0-9]+]], r3, [r1] -; CHECK: cmpeq [[STATUS]], #0 -; CHECK: bne [[DONE:LBB[0-9]+_[0-9]+]] -; CHECK: dmb ish -; CHECK: mov r0, #1 -; CHECK: [[DONE]]: -; CHECK: bx lr +; CHECK-NEXT: BB#0: +; CHECK-NEXT: dmb ish +; CHECK-NEXT: ldrex [[LOADED:r[0-9]+]], [r1] +; CHECK-NEXT: cmp [[LOADED]], r2 +; CHECK-NEXT: bne [[LDFAILBB:LBB[0-9]+_[0-9]+]] +; CHECK-NEXT: BB#1: +; CHECK-NEXT: strex [[SUCCESS:r[0-9]+]], r3, [r1] +; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: cmp [[SUCCESS]], #0 +; CHECK-NEXT: bxne lr +; CHECK-NEXT: dmb ish +; CHECK-NEXT: mov r0, #1 +; CHECK-NEXT: bx lr +; CHECK-NEXT: [[LDFAILBB]]: +; CHECK-NEXT: clrex +; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: bx lr ret i1 %success } Index: test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll =================================================================== --- test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll +++ test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll @@ -229,7 +229,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 @@ -241,6 +241,10 @@ ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[DONE]] @@ -263,7 +267,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 @@ -275,6 +279,10 @@ ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: dmb ; CHECK: br label %[[DONE]] @@ -296,7 +304,7 @@ ; CHECK: [[LOOP]]: ; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr) @@ -307,6 +315,10 @@ ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[DONE]] @@ -335,7 +347,7 @@ ; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 ; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 @@ -350,6 +362,10 @@ ; CHECK-NOT: dmb ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: dmb ; CHECK: br label %[[DONE]] Index: test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll =================================================================== --- test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll +++ test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll @@ -91,7 +91,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i8(i8* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 @@ -103,6 +103,10 @@ ; CHECK-NOT: fence_cst ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: fence_cst ; CHECK: br label %[[DONE]] @@ -125,7 +129,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 @@ -137,6 +141,10 @@ ; CHECK-NOT: fence ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: fence ; CHECK: br label %[[DONE]] @@ -158,7 +166,7 @@ ; CHECK: [[LOOP]]: ; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr) @@ -169,6 +177,10 @@ ; CHECK-NOT: fence_cst ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: fence_cst ; CHECK: br label %[[DONE]] @@ -197,7 +209,7 @@ ; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 ; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 @@ -212,6 +224,10 @@ ; CHECK-NOT: fence_cst ; CHECK: br label %[[DONE:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK-NEXT: call void @llvm.arm.clrex() +; CHECK-NEXT: br label %[[FAILURE_BB:.*]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: fence_cst ; CHECK: br label %[[DONE]] Index: test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll =================================================================== --- test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll +++ test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll @@ -9,17 +9,21 @@ ; CHECK: [[START]]: ; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr) ; CHECK: [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0 -; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB]] +; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]] ; CHECK: [[SUCCESS_BB]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[END:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK: call void @llvm.arm.clrex() +; CHECK: br label %[[FAILURE_BB]] + ; CHECK: [[FAILURE_BB]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[END]] @@ -41,7 +45,7 @@ ; CHECK: [[START]]: ; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr) @@ -52,6 +56,10 @@ ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: br label %[[END:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK: call void @llvm.arm.clrex() +; CHECK: br label %[[FAILURE_BB]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: dmb ; CHECK: br label %[[END]] @@ -73,7 +81,7 @@ ; CHECK: [[START]]: ; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr) @@ -84,6 +92,10 @@ ; CHECK-NOT: dmb ; CHECK: br label %[[END:.*]] +; CHECK: [[NO_STORE_BB]]: +; CHECK: call void @llvm.arm.clrex() +; CHECK: br label %[[FAILURE_BB]] + ; CHECK: [[FAILURE_BB]]: ; CHECK-NOT: dmb ; CHECK: br label %[[END]]