Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -1529,6 +1529,14 @@ llvm_unreachable("Custom atomicrmw expansion unimplemented on this target"); } + /// Emits a custom expansion of the AtomicCmpXchg. The implementer has + /// responsibility for replacing the passed instruction. Returns the + /// appropriately shifted/masked oldval. + virtual Value *emitAtomicCmpXchg(IRBuilder<> &Builder, + AtomicCmpXchgInst *CI) const { + llvm_unreachable("Custom atomicrmw expansion unimplemented on this target"); + } + /// Inserts in the IR a target-specific intrinsic specifying a fence. /// It is called by AtomicExpandPass before expanding an /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad @@ -1605,11 +1613,11 @@ return AtomicExpansionKind::None; } - /// Returns true if the given atomic cmpxchg should be expanded by the - /// IR-level AtomicExpand pass into a load-linked/store-conditional sequence - /// (through emitLoadLinked() and emitStoreConditional()). - virtual bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { - return false; + /// Returns how the given atomic cmpxchg should be expanded by the IR-level + /// AtomicExpand pass. + virtual AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { + return AtomicExpansionKind::None; } /// Returns how the IR-level AtomicExpand pass should expand the given Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -96,6 +96,7 @@ AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp, CreateCmpXchgInstFun CreateCmpXchg); + bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI); bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); bool isIdempotentRMW(AtomicRMWInst *AI); @@ -258,7 +259,9 @@ isAcquireOrStronger(RMWI->getOrdering()))) { FenceOrdering = RMWI->getOrdering(); RMWI->setOrdering(AtomicOrdering::Monotonic); - } else if (CASI && !TLI->shouldExpandAtomicCmpXchgInIR(CASI) && + } else if (CASI && + TLI->shouldExpandAtomicCmpXchgInIR(CASI) == + TargetLoweringBase::AtomicExpansionKind::None && (isReleaseOrStronger(CASI->getSuccessOrdering()) || isAcquireOrStronger(CASI->getSuccessOrdering()))) { // If a compare and swap is lowered to LL/SC, we can do smarter fence @@ -322,16 +325,7 @@ MadeChange = true; } - unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; - unsigned ValueSize = getAtomicOpSize(CASI); - if (ValueSize < MinCASSize) { - assert(!TLI->shouldExpandAtomicCmpXchgInIR(CASI) && - "MinCmpXchgSizeInBits not yet supported for LL/SC expansions."); - expandPartwordCmpXchg(CASI); - } else { - if (TLI->shouldExpandAtomicCmpXchgInIR(CASI)) - MadeChange |= expandAtomicCmpXchg(CASI); - } + MadeChange |= tryExpandAtomicCmpXchg(CASI); } } return MadeChange; @@ -1281,6 +1275,30 @@ return NewLoaded; } +bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) { + unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; + unsigned ValueSize = getAtomicOpSize(CI); + + switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) { + default: + llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg"); + case TargetLoweringBase::AtomicExpansionKind::None: + if (ValueSize < MinCASSize) + expandPartwordCmpXchg(CI); + return false; + case TargetLoweringBase::AtomicExpansionKind::LLSC: { + assert(ValueSize >= MinCASSize && + "MinCmpXchgSizeInBits not yet supported for LL/SC expansions."); + return expandAtomicCmpXchg(CI); + } + case TargetLoweringBase::AtomicExpansionKind::Custom: { + IRBuilder<> Builder(CI); + TLI->emitAtomicCmpXchg(Builder, CI); + return true; + } + } +} + // Note: This function is exposed externally by AtomicExpandUtils.h bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) { Index: lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.h +++ lib/Target/AArch64/AArch64ISelLowering.h @@ -388,7 +388,8 @@ TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override; - bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + TargetLoweringBase::AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; bool useLoadStackGuardNode() const override; TargetLoweringBase::LegalizeTypeAction Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -11230,16 +11230,20 @@ return (Subtarget->hasLSE() && Size < 128) ? AtomicExpansionKind::None : AtomicExpansionKind::LLSC; } -bool AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR( +TargetLowering::AtomicExpansionKind +AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR( AtomicCmpXchgInst *AI) const { // If subtarget has LSE, leave cmpxchg intact for codegen. - if (Subtarget->hasLSE()) return false; + if (Subtarget->hasLSE()) + return AtomicExpansionKind::None; // At -O0, fast-regalloc cannot cope with the live vregs necessary to // implement cmpxchg without spilling. If the address being exchanged is also // on the stack and close enough to the spill slot, this can lead to a // situation where the monitor always gets cleared and the atomic operation // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. - return getTargetMachine().getOptLevel() != 0; + if (getTargetMachine().getOptLevel() == 0) + return AtomicExpansionKind::None; + return AtomicExpansionKind::LLSC; } Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, Index: lib/Target/ARM/ARMISelLowering.h =================================================================== --- lib/Target/ARM/ARMISelLowering.h +++ lib/Target/ARM/ARMISelLowering.h @@ -535,7 +535,8 @@ bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override; - bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + TargetLoweringBase::AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; bool useLoadStackGuardNode() const override; Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -14320,16 +14320,18 @@ : AtomicExpansionKind::None; } -bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR( - AtomicCmpXchgInst *AI) const { +TargetLowering::AtomicExpansionKind +ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { // At -O0, fast-regalloc cannot cope with the live vregs necessary to // implement cmpxchg without spilling. If the address being exchanged is also // on the stack and close enough to the spill slot, this can lead to a // situation where the monitor always gets cleared and the atomic operation // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. - bool hasAtomicCmpXchg = + bool HasAtomicCmpXchg = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); - return getTargetMachine().getOptLevel() != 0 && hasAtomicCmpXchg; + if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg) + return AtomicExpansionKind::LLSC; + return AtomicExpansionKind::None; } bool ARMTargetLowering::shouldInsertFencesForAtomic( Index: lib/Target/Hexagon/HexagonISelLowering.h =================================================================== --- lib/Target/Hexagon/HexagonISelLowering.h +++ lib/Target/Hexagon/HexagonISelLowering.h @@ -305,7 +305,8 @@ Value *Addr, AtomicOrdering Ord) const override; AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; - bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override { Index: lib/Target/Hexagon/HexagonISelLowering.cpp =================================================================== --- lib/Target/Hexagon/HexagonISelLowering.cpp +++ lib/Target/Hexagon/HexagonISelLowering.cpp @@ -3152,9 +3152,12 @@ return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64; } -bool HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( - AtomicCmpXchgInst *AI) const { +TargetLowering::AtomicExpansionKind +HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( + AtomicCmpXchgInst *AI) const { const DataLayout &DL = AI->getModule()->getDataLayout(); unsigned Size = DL.getTypeStoreSize(AI->getCompareOperand()->getType()); - return Size >= 4 && Size <= 8; + if (Size >= 4 && Size <= 8) + return AtomicExpansionKind::LLSC; + return AtomicExpansionKind::None; }