Index: llvm/include/llvm/Transforms/Utils/LowerAtomic.h =================================================================== --- llvm/include/llvm/Transforms/Utils/LowerAtomic.h +++ llvm/include/llvm/Transforms/Utils/LowerAtomic.h @@ -14,9 +14,11 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H #define LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H +#include "llvm/IR/Instructions.h" + namespace llvm { -class AtomicCmpXchgInst; -class AtomicRMWInst; + +class IRBuilderBase; /// Convert the given Cmpxchg into primitive load and compare. bool lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI); @@ -25,6 +27,11 @@ /// assuming that doing so is legal. Return true if the lowering /// succeeds. bool lowerAtomicRMWInst(AtomicRMWInst *RMWI); + +/// Emit IR to implement the given atomicrmw operation on values in registers, +/// returning the new value. +Value *buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, + Value *Loaded, Value *Inc); } #endif // LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H Index: llvm/lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- llvm/lib/CodeGen/AtomicExpandPass.cpp +++ llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -544,47 +544,6 @@ NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy); } -/// Emit IR to implement the given atomicrmw operation on values in registers, -/// returning the new value. -static Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, - Value *Loaded, Value *Inc) { - Value *NewVal; - switch (Op) { - case AtomicRMWInst::Xchg: - return Inc; - case AtomicRMWInst::Add: - return Builder.CreateAdd(Loaded, Inc, "new"); - case AtomicRMWInst::Sub: - return Builder.CreateSub(Loaded, Inc, "new"); - case AtomicRMWInst::And: - return Builder.CreateAnd(Loaded, Inc, "new"); - case AtomicRMWInst::Nand: - return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new"); - case AtomicRMWInst::Or: - return Builder.CreateOr(Loaded, Inc, "new"); - case AtomicRMWInst::Xor: - return Builder.CreateXor(Loaded, Inc, "new"); - case AtomicRMWInst::Max: - NewVal = Builder.CreateICmpSGT(Loaded, Inc); - return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); - case AtomicRMWInst::Min: - NewVal = Builder.CreateICmpSLE(Loaded, Inc); - return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); - case AtomicRMWInst::UMax: - NewVal = Builder.CreateICmpUGT(Loaded, Inc); - return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); - case AtomicRMWInst::UMin: - NewVal = Builder.CreateICmpULE(Loaded, Inc); - return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); - case AtomicRMWInst::FAdd: - return Builder.CreateFAdd(Loaded, Inc, "new"); - case AtomicRMWInst::FSub: - return Builder.CreateFSub(Loaded, Inc, "new"); - default: - llvm_unreachable("Unknown atomic op"); - } -} - bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { LLVMContext &Ctx = AI->getModule()->getContext(); TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(AI); @@ -599,8 +558,8 @@ TargetLoweringBase::AtomicExpansionKind::LLSC); } else { auto PerformOp = [&](IRBuilder<> &Builder, Value *Loaded) { - return performAtomicOp(AI->getOperation(), Builder, Loaded, - AI->getValOperand()); + return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded, + AI->getValOperand()); }; expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(), AI->getAlign(), AI->getOrdering(), PerformOp); @@ -810,7 +769,7 @@ case AtomicRMWInst::Sub: case AtomicRMWInst::Nand: { // The other arithmetic ops need to be masked into place. - Value *NewVal = performAtomicOp(Op, Builder, Loaded, Shifted_Inc); + Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded, Shifted_Inc); Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask); Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked); @@ -824,7 +783,7 @@ // truncate down to the original size, and expand out again after // doing the operation. Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV); - Value *NewVal = performAtomicOp(Op, Builder, Loaded_Extract, Inc); + Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded_Extract, Inc); Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV); return FinalVal; } @@ -1558,8 +1517,8 @@ Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(), AI->getOrdering(), AI->getSyncScopeID(), [&](IRBuilder<> &Builder, Value *Loaded) { - return performAtomicOp(AI->getOperation(), Builder, Loaded, - AI->getValOperand()); + return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded, + AI->getValOperand()); }, CreateCmpXchg); Index: llvm/lib/Transforms/Utils/LowerAtomic.cpp =================================================================== --- llvm/lib/Transforms/Utils/LowerAtomic.cpp +++ llvm/lib/Transforms/Utils/LowerAtomic.cpp @@ -40,60 +40,53 @@ return true; } -bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { - IRBuilder<> Builder(RMWI); - Value *Ptr = RMWI->getPointerOperand(); - Value *Val = RMWI->getValOperand(); - - LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); - Value *Res = nullptr; - - switch (RMWI->getOperation()) { - default: llvm_unreachable("Unexpected RMW operation"); +Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, + IRBuilderBase &Builder, Value *Loaded, + Value *Inc) { + Value *NewVal; + switch (Op) { case AtomicRMWInst::Xchg: - Res = Val; - break; + return Inc; case AtomicRMWInst::Add: - Res = Builder.CreateAdd(Orig, Val); - break; + return Builder.CreateAdd(Loaded, Inc, "new"); case AtomicRMWInst::Sub: - Res = Builder.CreateSub(Orig, Val); - break; + return Builder.CreateSub(Loaded, Inc, "new"); case AtomicRMWInst::And: - Res = Builder.CreateAnd(Orig, Val); - break; + return Builder.CreateAnd(Loaded, Inc, "new"); case AtomicRMWInst::Nand: - Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val)); - break; + return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new"); case AtomicRMWInst::Or: - Res = Builder.CreateOr(Orig, Val); - break; + return Builder.CreateOr(Loaded, Inc, "new"); case AtomicRMWInst::Xor: - Res = Builder.CreateXor(Orig, Val); - break; + return Builder.CreateXor(Loaded, Inc, "new"); case AtomicRMWInst::Max: - Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val), - Val, Orig); - break; + NewVal = Builder.CreateICmpSGT(Loaded, Inc); + return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); case AtomicRMWInst::Min: - Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val), - Orig, Val); - break; + NewVal = Builder.CreateICmpSLE(Loaded, Inc); + return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); case AtomicRMWInst::UMax: - Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val), - Val, Orig); - break; + NewVal = Builder.CreateICmpUGT(Loaded, Inc); + return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); case AtomicRMWInst::UMin: - Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val), - Orig, Val); - break; + NewVal = Builder.CreateICmpULE(Loaded, Inc); + return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); case AtomicRMWInst::FAdd: - Res = Builder.CreateFAdd(Orig, Val); - break; + return Builder.CreateFAdd(Loaded, Inc, "new"); case AtomicRMWInst::FSub: - Res = Builder.CreateFSub(Orig, Val); - break; + return Builder.CreateFSub(Loaded, Inc, "new"); + default: + llvm_unreachable("Unknown atomic op"); } +} + +bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { + IRBuilder<> Builder(RMWI); + Value *Ptr = RMWI->getPointerOperand(); + Value *Val = RMWI->getValOperand(); + + LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); + Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val); Builder.CreateStore(Res, Ptr); RMWI->replaceAllUsesWith(Orig); RMWI->eraseFromParent();