Index: include/llvm/Target/TargetLowering.h =================================================================== --- include/llvm/Target/TargetLowering.h +++ include/llvm/Target/TargetLowering.h @@ -961,9 +961,10 @@ /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad. /// RMW and CmpXchg set both IsStore and IsLoad to true. /// Backends with !getInsertFencesForAtomic() should keep a no-op here. - virtual void emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, + virtual Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const { assert(!getInsertFencesForAtomic()); + return nullptr; } /// Inserts in the IR a target-specific intrinsic specifying a fence. @@ -971,9 +972,10 @@ /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad. /// RMW and CmpXchg set both IsStore and IsLoad to true. /// Backends with !getInsertFencesForAtomic() should keep a no-op here. - virtual void emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, + virtual Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const { assert(!getInsertFencesForAtomic()); + return nullptr; } /// Returns true if the given (atomic) store should be expanded by the Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -41,6 +41,8 @@ bool runOnFunction(Function &F) override; private: + bool bracketInstWithFences(Instruction *I, AtomicOrdering Order, + bool IsStore, bool IsLoad); bool expandAtomicLoad(LoadInst *LI); bool expandAtomicStore(StoreInst *SI); bool expandAtomicRMW(AtomicRMWInst *AI); @@ -80,10 +82,42 @@ auto SI = dyn_cast(I); auto RMWI = dyn_cast(I); auto CASI = dyn_cast(I); - assert((LI || SI || RMWI || CASI || isa(I)) && "Unknown atomic instruction"); + auto FenceOrdering = Monotonic; + bool IsStore, IsLoad; + if (TargetLowering->getInsertFencesForAtomic()) { + if (LI && isAtLeastAcquire(LI->getOrdering())) { + FenceOrdering = LI->getOrdering(); + LI->setOrdering(Monotonic); + IsStore = false; + IsLoad = true; + } else if (SI && isAtLeastRelease(SI->getOrdering())) { + FenceOrdering = SI->getOrdering(); + SI->setOrdering(Monotonic); + IsStore = true; + IsLoad = false; + } else if (RMWI && !TargetLowering->shouldExpandAtomicRMWInIR(RMWI)) { + // We do not bother calling bracketInstWithFences if the RMW is already + // set for expansion, as the expansion already inserts whatever fence is + // required. + FenceOrdering = RMWI->getOrdering(); + RMWI->setOrdering(Monotonic); + IsStore = IsLoad = true; + } else if (CASI && !TargetLowering->hasLoadLinkedStoreConditional()) { + // Same as above. + FenceOrdering = CASI->getSuccessOrdering(); + CASI->setSuccessOrdering(Monotonic); + CASI->setFailureOrdering(Monotonic); + IsStore = IsLoad = true; + } + + if (FenceOrdering != Monotonic) { + MadeChange |= bracketInstWithFences(I, FenceOrdering, IsStore, IsLoad); + } + } + if (LI && TargetLowering->shouldExpandAtomicLoadInIR(LI)) { MadeChange |= expandAtomicLoad(LI); } else if (SI && TargetLowering->shouldExpandAtomicStoreInIR(SI)) { @@ -97,6 +131,31 @@ return MadeChange; } +bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order, + bool IsStore, bool IsLoad) { + IRBuilder<> Builder(I); + + auto LeadingFence = + TM->getSubtargetImpl()->getTargetLowering()->emitLeadingFence( + Builder, Order, IsStore, IsLoad); + + auto TrailingFence = + TM->getSubtargetImpl()->getTargetLowering()->emitTrailingFence( + Builder, Order, IsStore, IsLoad); + // The trailing fence is emitted before the instruction instead of after + // because there is no easy way of setting Builder insertion point after + // an instruction. So we must erase it from the BB, and insert it back + // in the right place. + // We have a guard here because not every atomic operation generates a + // trailing fence. + if (TrailingFence) { + TrailingFence->removeFromParent(); + TrailingFence->insertAfter(I); + } + + return (LeadingFence || TrailingFence); +} + bool AtomicExpand::expandAtomicLoad(LoadInst *LI) { auto TLI = TM->getSubtargetImpl()->getTargetLowering(); // If getInsertFencesForAtomic() returns true, then the target does not want Index: lib/Target/ARM/ARMISelLowering.h =================================================================== --- lib/Target/ARM/ARMISelLowering.h +++ lib/Target/ARM/ARMISelLowering.h @@ -398,9 +398,9 @@ Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override; - void emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, + Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const override; - void emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, + Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const override; bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override; Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -10984,19 +10984,19 @@ bool ARMTargetLowering::hasLoadLinkedStoreConditional() const { return true; } -static void makeDMB(IRBuilder<> &Builder, ARM_MB::MemBOpt Domain) { +static Instruction* makeDMB(IRBuilder<> &Builder, ARM_MB::MemBOpt Domain) { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); Function *DMB = llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_dmb); Constant *CDomain = Builder.getInt32(Domain); - Builder.CreateCall(DMB, CDomain); + return Builder.CreateCall(DMB, CDomain); } // Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html -void ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder, +Instruction* ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const { if (!getInsertFencesForAtomic()) - return; + return nullptr; switch (Ord) { case NotAtomic: @@ -11004,27 +11004,26 @@ llvm_unreachable("Invalid fence: unordered/non-atomic"); case Monotonic: case Acquire: - return; // Nothing to do + return nullptr; // Nothing to do case SequentiallyConsistent: if (!IsStore) - return; // Nothing to do - /*FALLTHROUGH*/ + return nullptr; // Nothing to do + /*FALLTHROUGH*/ case Release: case AcquireRelease: if (Subtarget->isSwift()) - makeDMB(Builder, ARM_MB::ISHST); + return makeDMB(Builder, ARM_MB::ISHST); // FIXME: add a comment with a link to documentation justifying this. else - makeDMB(Builder, ARM_MB::ISH); - return; + return makeDMB(Builder, ARM_MB::ISH); } } -void ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder, +Instruction* ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord, bool IsStore, bool IsLoad) const { if (!getInsertFencesForAtomic()) - return; + return nullptr; switch (Ord) { case NotAtomic: @@ -11032,12 +11031,11 @@ llvm_unreachable("Invalid fence: unordered/not-atomic"); case Monotonic: case Release: - return; // Nothing to do + return nullptr; // Nothing to do case Acquire: case AcquireRelease: - case SequentiallyConsistent: - makeDMB(Builder, ARM_MB::ISH); - return; + case SequentiallyConsistent: + return makeDMB(Builder, ARM_MB::ISH); } }