Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -1574,6 +1574,15 @@ llvm_unreachable("Masked atomicrmw expansion unimplemented on this target"); } + /// Perform a masked cmpxchg using a target-specific intrinsic. This + /// represents the core LL/SC loop which will be lowered at a late stage by + /// the backend. + virtual Value *emitMaskedAtomicCmpXchgIntrinsic( + IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, + Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { + llvm_unreachable("Masked cmpxchg expansion unimplemented on this target"); + } + /// Inserts in the IR a target-specific intrinsic specifying a fence. /// It is called by AtomicExpandPass before expanding an /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad @@ -1650,11 +1659,11 @@ return AtomicExpansionKind::None; } - /// Returns true if the given atomic cmpxchg should be expanded by the - /// IR-level AtomicExpand pass into a load-linked/store-conditional sequence - /// (through emitLoadLinked() and emitStoreConditional()). - virtual bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { - return false; + /// Returns how the given atomic cmpxchg should be expanded by the IR-level + /// AtomicExpand pass. + virtual AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { + return AtomicExpansionKind::None; } /// Returns how the IR-level AtomicExpand pass should expand the given Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -98,6 +98,7 @@ AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp, CreateCmpXchgInstFun CreateCmpXchg); + bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI); bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); bool isIdempotentRMW(AtomicRMWInst *RMWI); @@ -260,7 +261,9 @@ isAcquireOrStronger(RMWI->getOrdering()))) { FenceOrdering = RMWI->getOrdering(); RMWI->setOrdering(AtomicOrdering::Monotonic); - } else if (CASI && !TLI->shouldExpandAtomicCmpXchgInIR(CASI) && + } else if (CASI && + TLI->shouldExpandAtomicCmpXchgInIR(CASI) == + TargetLoweringBase::AtomicExpansionKind::None && (isReleaseOrStronger(CASI->getSuccessOrdering()) || isAcquireOrStronger(CASI->getSuccessOrdering()))) { // If a compare and swap is lowered to LL/SC, we can do smarter fence @@ -334,16 +337,7 @@ MadeChange = true; } - unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; - unsigned ValueSize = getAtomicOpSize(CASI); - if (ValueSize < MinCASSize) { - assert(!TLI->shouldExpandAtomicCmpXchgInIR(CASI) && - "MinCmpXchgSizeInBits not yet supported for LL/SC expansions."); - expandPartwordCmpXchg(CASI); - } else { - if (TLI->shouldExpandAtomicCmpXchgInIR(CASI)) - MadeChange |= expandAtomicCmpXchg(CASI); - } + MadeChange |= tryExpandAtomicCmpXchg(CASI); } } return MadeChange; @@ -1355,6 +1349,28 @@ return NewLoaded; } +bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) { + unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; + unsigned ValueSize = getAtomicOpSize(CI); + + switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) { + default: + llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg"); + case TargetLoweringBase::AtomicExpansionKind::None: + if (ValueSize < MinCASSize) + expandPartwordCmpXchg(CI); + return false; + case TargetLoweringBase::AtomicExpansionKind::LLSC: { + assert(ValueSize >= MinCASSize && + "MinCmpXchgSizeInBits not yet supported for LL/SC expansions."); + return expandAtomicCmpXchg(CI); + } + case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: + llvm_unreachable( + "MaskedIntrinsic expansion of cmpxhg not yet implemented"); + } +} + // Note: This function is exposed externally by AtomicExpandUtils.h bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) { Index: lib/Target/AArch64/AArch64ISelLowering.h =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.h +++ lib/Target/AArch64/AArch64ISelLowering.h @@ -390,7 +390,8 @@ TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override; - bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + TargetLoweringBase::AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; bool useLoadStackGuardNode() const override; TargetLoweringBase::LegalizeTypeAction Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -11377,16 +11377,20 @@ return (Subtarget->hasLSE() && Size < 128) ? AtomicExpansionKind::None : AtomicExpansionKind::LLSC; } -bool AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR( +TargetLowering::AtomicExpansionKind +AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR( AtomicCmpXchgInst *AI) const { // If subtarget has LSE, leave cmpxchg intact for codegen. - if (Subtarget->hasLSE()) return false; + if (Subtarget->hasLSE()) + return AtomicExpansionKind::None; // At -O0, fast-regalloc cannot cope with the live vregs necessary to // implement cmpxchg without spilling. If the address being exchanged is also // on the stack and close enough to the spill slot, this can lead to a // situation where the monitor always gets cleared and the atomic operation // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. - return getTargetMachine().getOptLevel() != 0; + if (getTargetMachine().getOptLevel() == 0) + return AtomicExpansionKind::None; + return AtomicExpansionKind::LLSC; } Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, Index: lib/Target/ARM/ARMISelLowering.h =================================================================== --- lib/Target/ARM/ARMISelLowering.h +++ lib/Target/ARM/ARMISelLowering.h @@ -538,7 +538,8 @@ bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override; - bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + TargetLoweringBase::AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; bool useLoadStackGuardNode() const override; Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -14561,16 +14561,18 @@ : AtomicExpansionKind::None; } -bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR( - AtomicCmpXchgInst *AI) const { +TargetLowering::AtomicExpansionKind +ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const { // At -O0, fast-regalloc cannot cope with the live vregs necessary to // implement cmpxchg without spilling. If the address being exchanged is also // on the stack and close enough to the spill slot, this can lead to a // situation where the monitor always gets cleared and the atomic operation // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. - bool hasAtomicCmpXchg = + bool HasAtomicCmpXchg = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps(); - return getTargetMachine().getOptLevel() != 0 && hasAtomicCmpXchg; + if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg) + return AtomicExpansionKind::LLSC; + return AtomicExpansionKind::None; } bool ARMTargetLowering::shouldInsertFencesForAtomic( Index: lib/Target/Hexagon/HexagonISelLowering.h =================================================================== --- lib/Target/Hexagon/HexagonISelLowering.h +++ lib/Target/Hexagon/HexagonISelLowering.h @@ -311,7 +311,8 @@ Value *Addr, AtomicOrdering Ord) const override; AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override; bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override; - bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; + AtomicExpansionKind + shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override { Index: lib/Target/Hexagon/HexagonISelLowering.cpp =================================================================== --- lib/Target/Hexagon/HexagonISelLowering.cpp +++ lib/Target/Hexagon/HexagonISelLowering.cpp @@ -3214,9 +3214,12 @@ return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64; } -bool HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( - AtomicCmpXchgInst *AI) const { +TargetLowering::AtomicExpansionKind +HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR( + AtomicCmpXchgInst *AI) const { const DataLayout &DL = AI->getModule()->getDataLayout(); unsigned Size = DL.getTypeStoreSize(AI->getCompareOperand()->getType()); - return Size >= 4 && Size <= 8; + if (Size >= 4 && Size <= 8) + return AtomicExpansionKind::LLSC; + return AtomicExpansionKind::None; }