Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -1481,11 +1481,24 @@ /// Returns the maximum atomic operation size (in bits) supported by /// the backend. Atomic operations greater than this size (as well /// as ones that are not naturally aligned), will be expanded by - /// AtomicExpandPass into an __atomic_* library call. + /// AtomicExpandPass into an __atomic_* library call unless + /// shouldExpandAtomicToLibCall is overridden. unsigned getMaxAtomicSizeInBitsSupported() const { return MaxAtomicSizeInBitsSupported; } + /// Indicates whether this atomic should be expanded to a libcall. Defaults to + /// expanding when the atomic is less than naturally aligned or its size is + /// greater than MaxAtomicSizeInBitsSupported. If the atomic isn't expanded + /// it will be passed through for target lowering. Targets overriding this + /// method should give the same result for atomics of the same size unless + /// the libcall is known to be lock-free, in order to avoid the invalid + /// mixing of inline lock-free atomic operations and locked libcalls. + virtual bool shouldExpandAtomicToLibCall(unsigned Size, unsigned Align, + Instruction *I) const { + return Align < Size || Size > getMaxAtomicSizeInBitsSupported() / 8; + } + /// Returns the size of the smallest cmpxchg or ll/sc instruction /// the backend supports. Any smaller operations are widened in /// AtomicExpandPass. Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -181,16 +181,6 @@ return DL.getTypeStoreSize(CASI->getCompareOperand()->getType()); } -// Determine if a particular atomic operation has a supported size, -// and is of appropriate alignment, to be passed through for target -// lowering. (Versus turning into a __atomic libcall) -template -static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) { - unsigned Size = getAtomicOpSize(I); - unsigned Align = getAtomicOpAlign(I); - return Align >= Size && Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8; -} - bool AtomicExpand::runOnFunction(Function &F) { auto *TPC = getAnalysisIfAvailable(); if (!TPC) @@ -221,25 +211,29 @@ // If the Size/Alignment is not supported, replace with a libcall. if (LI) { - if (!atomicSizeSupported(TLI, LI)) { + if (TLI->shouldExpandAtomicToLibCall(getAtomicOpSize(LI), + getAtomicOpAlign(LI), LI)) { expandAtomicLoadToLibcall(LI); MadeChange = true; continue; } } else if (SI) { - if (!atomicSizeSupported(TLI, SI)) { + if (TLI->shouldExpandAtomicToLibCall(getAtomicOpSize(SI), + getAtomicOpAlign(SI), SI)) { expandAtomicStoreToLibcall(SI); MadeChange = true; continue; } } else if (RMWI) { - if (!atomicSizeSupported(TLI, RMWI)) { + if (TLI->shouldExpandAtomicToLibCall(getAtomicOpSize(RMWI), + getAtomicOpAlign(RMWI), RMWI)) { expandAtomicRMWToLibcall(RMWI); MadeChange = true; continue; } } else if (CASI) { - if (!atomicSizeSupported(TLI, CASI)) { + if (TLI->shouldExpandAtomicToLibCall(getAtomicOpSize(CASI), + getAtomicOpAlign(CASI), CASI)) { expandAtomicCASToLibcall(CASI); MadeChange = true; continue;