Index: include/llvm/Analysis/BranchProbabilityInfo.h =================================================================== --- include/llvm/Analysis/BranchProbabilityInfo.h +++ include/llvm/Analysis/BranchProbabilityInfo.h @@ -115,6 +115,8 @@ return IsLikely ? (1u << 20) - 1 : 1; } + static MDNode *getBranchWeightsAtomicExpand(bool Reverse, LLVMContext &Ctx); + private: // Since we allow duplicate edges from one basic block to another, we use // a pair (PredBlock and an index in the successors) to specify an edge. Index: lib/Analysis/BranchProbabilityInfo.cpp =================================================================== --- lib/Analysis/BranchProbabilityInfo.cpp +++ lib/Analysis/BranchProbabilityInfo.cpp @@ -19,6 +19,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/Debug.h" @@ -630,6 +631,16 @@ << Weight << "\n"); } +MDNode *BranchProbabilityInfo::getBranchWeightsAtomicExpand(bool Reverse, + LLVMContext &Ctx) { + unsigned SuccessWeight = (1u << 20) - 1, FailureWeight = 1; + + if (Reverse) + std::swap(SuccessWeight, FailureWeight); + + return MDBuilder(Ctx).createBranchWeights(SuccessWeight, FailureWeight); +} + /// Get an edge's probability, relative to other out-edges from Src. BranchProbability BranchProbabilityInfo:: getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const { Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -12,12 +12,14 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Target/TargetLowering.h" @@ -312,7 +314,9 @@ TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder); Value *TryAgain = Builder.CreateICmpNE( StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain"); - Builder.CreateCondBr(TryAgain, LoopBB, ExitBB); + MDNode *Weights = BranchProbabilityInfo::getBranchWeightsAtomicExpand( + true, F->getContext()); + Builder.CreateCondBr(TryAgain, LoopBB, ExitBB, Weights); Builder.SetInsertPoint(ExitBB, ExitBB->begin()); @@ -376,7 +380,9 @@ Loaded->addIncoming(NewLoaded, LoopBB); Value *Success = Builder.CreateExtractValue(Pair, 1, "success"); - Builder.CreateCondBr(Success, ExitBB, LoopBB); + MDNode *Weights = BranchProbabilityInfo::getBranchWeightsAtomicExpand( + false, F->getContext()); + Builder.CreateCondBr(Success, ExitBB, LoopBB, Weights); Builder.SetInsertPoint(ExitBB, ExitBB->begin()); @@ -451,7 +457,9 @@ // If the the cmpxchg doesn't actually need any ordering when it fails, we can // jump straight past that fence instruction (if it exists). - Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB); + MDNode *Weights = BranchProbabilityInfo::getBranchWeightsAtomicExpand( + false, F->getContext()); + Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB, Weights); Builder.SetInsertPoint(TryStoreBB); Value *StoreSuccess = TLI->emitStoreConditional( @@ -459,7 +467,7 @@ StoreSuccess = Builder.CreateICmpEQ( StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success"); Builder.CreateCondBr(StoreSuccess, SuccessBB, - CI->isWeak() ? FailureBB : LoopBB); + CI->isWeak() ? FailureBB : LoopBB, Weights); // Make sure later instructions don't get reordered with a fence if necessary. Builder.SetInsertPoint(SuccessBB); Index: test/CodeGen/AArch64/arm64-atomic.ll =================================================================== --- test/CodeGen/AArch64/arm64-atomic.ll +++ test/CodeGen/AArch64/arm64-atomic.ll @@ -47,9 +47,8 @@ define i64 @fetch_and_nand_64(i64* %p) { ; CHECK-LABEL: fetch_and_nand_64: -; CHECK: mov x[[ADDR:[0-9]+]], x0 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR]]] +; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR:[0-9]+]]] ; CHECK: mvn w[[TMP_REG:[0-9]+]], w[[DEST_REG]] ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], x[[TMP_REG]], #0xfffffffffffffff8 ; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] @@ -75,9 +74,8 @@ define i64 @fetch_and_or_64(i64* %p) { ; CHECK: fetch_and_or_64: -; CHECK: mov x[[ADDR:[0-9]+]], x0 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR]]] +; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR:[0-9]+]]] ; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], [[DEST_REG]], #0x7 ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] ; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]] Index: test/CodeGen/ARM/atomic-64bit.ll =================================================================== --- test/CodeGen/ARM/atomic-64bit.ll +++ test/CodeGen/ARM/atomic-64bit.ll @@ -174,10 +174,10 @@ ; CHECK-DAG: mov [[VAL1LO:r[0-9]+]], r1 ; CHECK-DAG: dmb {{ish$}} ; CHECK: ldrexd [[REG1:(r[0-9]?[02468])]], [[REG2:(r[0-9]?[13579])]] -; CHECK-LE-DAG: eor [[MISMATCH_LO:r[0-9]+]], [[REG1]], [[VAL1LO]] -; CHECK-LE-DAG: eor [[MISMATCH_HI:r[0-9]+]], [[REG2]], r2 -; CHECK-BE-DAG: eor [[MISMATCH_LO:r[0-9]+]], [[REG2]], r2 -; CHECK-BE-DAG: eor [[MISMATCH_HI:r[0-9]+]], [[REG1]], r1 +; CHECK-LE-DAG: eor [[MISMATCH_LO:lr|r[0-9]+]], [[REG1]], [[VAL1LO]] +; CHECK-LE-DAG: eor [[MISMATCH_HI:lr|r[0-9]+]], [[REG2]], r2 +; CHECK-BE-DAG: eor [[MISMATCH_LO:lr|r[0-9]+]], [[REG2]], r2 +; CHECK-BE-DAG: eor [[MISMATCH_HI:lr|r[0-9]+]], [[REG1]], r1 ; CHECK: orrs {{r[0-9]+}}, [[MISMATCH_LO]], [[MISMATCH_HI]] ; CHECK: bne ; CHECK: strexd {{[a-z0-9]+}}, {{r[0-9]?[02468]}}, {{r[0-9]?[13579]}} Index: test/CodeGen/ARM/cmpxchg-idioms.ll =================================================================== --- test/CodeGen/ARM/cmpxchg-idioms.ll +++ test/CodeGen/ARM/cmpxchg-idioms.ll @@ -6,21 +6,17 @@ ; CHECK: dmb ishst ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: -; CHECK: ldrex [[LOADED:r[0-9]+]], [r0] -; CHECK: cmp [[LOADED]], r1 -; CHECK: bne [[FAILED:LBB[0-9]+_[0-9]+]] - ; CHECK: strex [[STATUS:r[0-9]+]], {{r[0-9]+}}, [r0] ; CHECK: cmp [[STATUS]], #0 -; CHECK: bne [[LOOP]] +; CHECK: ittt eq +; CHECK: dmbeq ish +; CHECK: moveq r0, #1 +; CHECK: bxeq lr -; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}} -; CHECK: movs r0, #1 -; CHECK: dmb ish -; CHECK: bx lr +; CHECK: ldrex [[LOADED:r[0-9]+]], [r0] +; CHECK: cmp [[LOADED]], r1 +; CHECK: beq [[LOOP]] -; CHECK: [[FAILED]]: -; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}} ; CHECK: movs r0, #0 ; CHECK: dmb ish ; CHECK: bx lr @@ -37,27 +33,24 @@ ; CHECK: uxtb [[OLDBYTE:r[0-9]+]], r1 ; CHECK: dmb ishst + ; FIXME: eoreq and eor are redundant. Need to teach DAG combine that. ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: -; CHECK: ldrexb [[LOADED:r[0-9]+]], [r0] -; CHECK: cmp [[LOADED]], [[OLDBYTE]] -; CHECK: bne [[FAIL:LBB[0-9]+_[0-9]+]] - ; CHECK: strexb [[STATUS:r[0-9]+]], {{r[0-9]+}}, [r0] ; CHECK: cmp [[STATUS]], #0 -; CHECK: bne [[LOOP]] +; CHECK: itttt eq +; CHECK: dmbeq ish +; CHECK: moveq [[TMP:r[0-9]+]], #1 +; CHECK: eoreq r0, [[TMP]], #1 +; CHECK: bxeq lr - ; FIXME: this eor is redundant. Need to teach DAG combine that. -; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}} -; CHECK: movs [[TMP:r[0-9]+]], #1 -; CHECK: eor r0, [[TMP]], #1 -; CHECK: bx lr +; CHECK: ldrexb [[LOADED:r[0-9]+]], [r0] +; CHECK: cmp [[LOADED]], [[OLDBYTE]] +; CHECK: beq [[LOOP]] -; CHECK: [[FAIL]]: ; CHECK: movs [[TMP:r[0-9]+]], #0 ; CHECK: eor r0, [[TMP]], #1 ; CHECK: bx lr - %pair = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic %success = extractvalue { i8, i1 } %pair, 1 %failure = xor i1 %success, 1 @@ -70,20 +63,16 @@ ; CHECK: dmb ishst ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: -; CHECK: ldrex [[LOADED:r[0-9]+]], [r0] -; CHECK: cmp [[LOADED]], r1 -; CHECK: bne [[FAILED:LBB[0-9]+_[0-9]+]] - ; CHECK: strex [[STATUS:r[0-9]+]], r2, [r0] ; CHECK: cmp [[STATUS]], #0 -; CHECK: bne [[LOOP]] +; CHECK: itt eq +; CHECK: dmbeq ish +; CHECK: beq.w _bar -; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}} -; CHECK: dmb ish -; CHECK: b.w _bar +; CHECK: ldrex [[LOADED:r[0-9]+]], [r0] +; CHECK: cmp [[LOADED]], r1 +; CHECK: beq [[LOOP]] -; CHECK: [[FAILED]]: -; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}} ; CHECK: dmb ish ; CHECK: b.w _baz Index: test/CodeGen/X86/atomic128.ll =================================================================== --- test/CodeGen/X86/atomic128.ll +++ test/CodeGen/X86/atomic128.ll @@ -269,11 +269,11 @@ define void @atomic_store_seq_cst(i128* %p, i128 %in) { ; CHECK-LABEL: atomic_store_seq_cst: ; CHECK: movq %rdx, %rcx -; CHECK: movq %rsi, %rbx ; CHECK: movq (%rdi), %rax ; CHECK: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: +; CHECK-DAG: movq %rsi, %rbx ; CHECK: lock ; CHECK: cmpxchg16b (%rdi) ; CHECK: jne [[LOOP]] @@ -286,11 +286,11 @@ define void @atomic_store_release(i128* %p, i128 %in) { ; CHECK-LABEL: atomic_store_release: ; CHECK: movq %rdx, %rcx -; CHECK: movq %rsi, %rbx ; CHECK: movq (%rdi), %rax ; CHECK: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: +; CHECK: movq %rsi, %rbx ; CHECK: lock ; CHECK: cmpxchg16b (%rdi) ; CHECK: jne [[LOOP]] @@ -302,11 +302,11 @@ define void @atomic_store_relaxed(i128* %p, i128 %in) { ; CHECK-LABEL: atomic_store_relaxed: ; CHECK: movq %rdx, %rcx -; CHECK: movq %rsi, %rbx ; CHECK: movq (%rdi), %rax ; CHECK: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: +; CHECK: movq %rsi, %rbx ; CHECK: lock ; CHECK: cmpxchg16b (%rdi) ; CHECK: jne [[LOOP]] Index: test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll =================================================================== --- test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll +++ test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll @@ -10,7 +10,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0:[0-9]+]] ; CHECK: [[END]]: ; CHECK-NOT: dmb ; CHECK: ret i8 [[OLDVAL]] @@ -29,7 +29,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i16 [[OLDVAL]] @@ -46,7 +46,7 @@ ; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 [[NEWVAL]], i32* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i32 [[OLDVAL]] @@ -65,7 +65,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK-NOT: dmb ; CHECK: ret i8 [[OLDVAL]] @@ -85,7 +85,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i16 [[OLDVAL]] @@ -113,7 +113,7 @@ ; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]]) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i64 [[OLDVAL]] @@ -132,7 +132,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i8 [[OLDVAL]] @@ -152,7 +152,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i8 [[OLDVAL]] @@ -172,7 +172,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i8 [[OLDVAL]] @@ -192,7 +192,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i8 [[OLDVAL]] @@ -212,7 +212,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0]] ; CHECK: [[END]]: ; CHECK: call void @llvm.arm.dmb(i32 11) ; CHECK: ret i8 [[OLDVAL]] @@ -229,7 +229,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 @@ -263,7 +263,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 @@ -296,7 +296,7 @@ ; CHECK: [[LOOP]]: ; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr) @@ -335,7 +335,7 @@ ; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 ; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 @@ -362,3 +362,6 @@ %old = extractvalue { i64, i1 } %pairold, 0 ret i64 %old } + +; CHECK: ![[PROF0]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: ![[PROF1]] = !{!"branch_weights", i32 1048575, i32 1} Index: test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll =================================================================== --- test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll +++ test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll @@ -10,7 +10,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0:[0-9]+]] ; CHECK: [[END]]: ; CHECK-NOT: fence ; CHECK: ret i8 [[OLDVAL]] @@ -29,7 +29,7 @@ ; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i16(i32 [[NEWVAL32]], i16* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0:[0-9]+]] ; CHECK: [[END]]: ; CHECK-NOT: fence ; CHECK: ret i16 [[OLDVAL]] @@ -46,7 +46,7 @@ ; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 [[NEWVAL]], i32* %ptr) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0:[0-9]+]] ; CHECK: [[END]]: ; CHECK-NOT: fence ; CHECK: ret i32 [[OLDVAL]] @@ -74,7 +74,7 @@ ; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]]) ; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 -; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]], !prof ![[PROF0:[0-9]+]] ; CHECK: [[END]]: ; CHECK-NOT: fence ; CHECK: ret i64 [[OLDVAL]] @@ -91,7 +91,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i8(i8* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 @@ -125,7 +125,7 @@ ; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr) ; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 @@ -158,7 +158,7 @@ ; CHECK: [[LOOP]]: ; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr) @@ -197,7 +197,7 @@ ; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 ; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 @@ -224,3 +224,6 @@ %old = extractvalue { i64, i1 } %pairold, 0 ret i64 %old } + +; CHECK: ![[PROF0]] = !{!"branch_weights", i32 1, i32 1048575} +; CHECK: ![[PROF1]] = !{!"branch_weights", i32 1048575, i32 1} Index: test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll =================================================================== --- test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll +++ test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll @@ -9,7 +9,7 @@ ; CHECK: [[START]]: ; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr) @@ -41,12 +41,12 @@ ; CHECK: [[START]]: ; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr) ; CHECK: [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0 -; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB]] ; CHECK: [[SUCCESS_BB]]: ; CHECK: call void @llvm.arm.dmb(i32 11) @@ -73,12 +73,12 @@ ; CHECK: [[START]]: ; CHECK: [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr) ; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired -; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[FAILURE_BB:.*]], !prof ![[PROF1:[0-9]+]] ; CHECK: [[TRY_STORE]]: ; CHECK: [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr) ; CHECK: [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0 -; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]] +; CHECK: br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB]] ; CHECK: [[SUCCESS_BB]]: ; CHECK-NOT: dmb @@ -96,3 +96,5 @@ %oldval = extractvalue { i32, i1 } %pair, 0 ret i32 %oldval } + +; CHECK: ![[PROF1]] = !{!"branch_weights", i32 1048575, i32 1} Index: test/Transforms/AtomicExpand/X86/atomic32-weight.ll =================================================================== --- /dev/null +++ test/Transforms/AtomicExpand/X86/atomic32-weight.ll @@ -0,0 +1,17 @@ +; RUN: opt -S -o - -mtriple=i386-apple-ios -atomic-expand %s | FileCheck %s + +@sc32 = external global i32 + +; CHECK: br label %[[STARTBB:.*]] +; CHECK: [[STARTBB]]: +; CHECK: [[VAL_SUCCESS:%[0-9]+]] = cmpxchg +; CHECK: [[SUCCESS:%[a-z]+]] = extractvalue { i32, i1 } [[VAL_SUCCESS]], 1 +; CHECK: br i1 [[SUCCESS]], label %[[ENDBB:.*]], label %[[STARTBB]], !prof !0 +; CHECK: [[ENDBB]]: +; CHECK: !0 = !{!"branch_weights", i32 1048575, i32 1} + +define void @atomic_fetch_and32() nounwind { + %t2 = atomicrmw and i32* @sc32, i32 5 acquire + %t3 = atomicrmw and i32* @sc32, i32 %t2 acquire + ret void +} Index: test/Transforms/AtomicExpand/X86/lit.local.cfg =================================================================== --- /dev/null +++ test/Transforms/AtomicExpand/X86/lit.local.cfg @@ -0,0 +1,3 @@ +if not 'X86' in config.root.targets: + config.unsupported = True +