Index: include/llvm/CodeGen/Passes.h =================================================================== --- include/llvm/CodeGen/Passes.h +++ include/llvm/CodeGen/Passes.h @@ -349,6 +349,8 @@ /// List of target independent CodeGen pass IDs. namespace llvm { + FunctionPass *createAtomicExpandLoadLinkedPass(const TargetMachine *TM); + /// \brief Create a basic TargetTransformInfo analysis pass. /// /// This pass implements the target transform info analysis using the target @@ -374,6 +376,9 @@ /// matching during instruction selection. FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = 0); + /// AtomicExpandLoadLinkedID -- FIXME + extern char &AtomicExpandLoadLinkedID; + /// MachineLoopInfo - This pass is a loop analysis pass. extern char &MachineLoopInfoID; Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -71,6 +71,7 @@ void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlwaysInlinerPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); +void initializeAtomicExpandLoadLinkedPass(PassRegistry&); void initializeSampleProfileLoaderPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAliasAnalysisPass(PassRegistry&); Index: include/llvm/Target/TargetLowering.h =================================================================== --- include/llvm/Target/TargetLowering.h +++ include/llvm/Target/TargetLowering.h @@ -31,6 +31,7 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/InlineAsm.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Target/TargetCallingConv.h" #include "llvm/Target/TargetMachine.h" @@ -897,6 +898,35 @@ /// @} //===--------------------------------------------------------------------===// + /// \name Helpers for load-linked/store-conditional atomic expansion. + /// @{ + + /// Perform a load-linked operation on Addr, returning a "Value *" with the + /// corresponding pointee type. This may entail some non-trivial operations to + /// truncate or reconstruct types that will be illegal in the backend. See + /// ARMISelLowering for an example implementation. + virtual Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr, + AtomicOrdering Ord) const { + llvm_unreachable("Load linked unimplemented on this target"); + } + + /// Perform a store-conditional operation to Addr. Return the status of the + /// store. This should be 0 if the store succeeded, non-zero otherwise. + virtual Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, + Value *Addr, AtomicOrdering Ord) const { + llvm_unreachable("Store conditional unimplemented on this target"); + } + + /// Return true if the given (atomic) instruction should be expanded by the + /// IR-level AtomicExpandLoadLinked pass into a loop involving + /// load-linked/store-conditional pairs. Atomic stores will be expanded in the + /// same way as "atomic xchg" operations which ignore their output if needed. + virtual bool shouldExpandAtomicInIR(Instruction *Inst) const { + return false; + } + + + //===--------------------------------------------------------------------===// // TargetLowering Configuration Methods - These methods should be invoked by // the derived class constructor to configure this object for the target. // Index: lib/CodeGen/AtomicExpandLoadLinkedPass.cpp =================================================================== --- /dev/null +++ lib/CodeGen/AtomicExpandLoadLinkedPass.cpp @@ -0,0 +1,336 @@ +//===-- AtomicExpandLoadLinkedPass.cpp - Expand atomic instructions -------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains a pass (at IR level) to replace atomic instructions with +// appropriate (intrinsic-based) ldrex/strex loops. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "arm-atomic-expand" +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Debug.h" +#include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetMachine.h" +using namespace llvm; + +namespace { + class AtomicExpandLoadLinked : public FunctionPass { + const TargetLowering *TLI; + public: + static char ID; // Pass identification, replacement for typeid + explicit AtomicExpandLoadLinked(const TargetMachine *TM = 0) + : FunctionPass(ID), TLI(TM ? TM->getTargetLowering() : 0) { + initializeAtomicExpandLoadLinkedPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + bool expandAtomicInsts(Function &F); + + bool expandAtomicLoad(LoadInst *LI); + bool expandAtomicStore(StoreInst *LI); + bool expandAtomicRMW(AtomicRMWInst *AI); + bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); + + AtomicOrdering insertLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord); + void insertTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord); + }; +} + +char AtomicExpandLoadLinked::ID = 0; +char &llvm::AtomicExpandLoadLinkedID = AtomicExpandLoadLinked::ID; + +static void *initializeAtomicExpandLoadLinkedPassOnce(PassRegistry &Registry) { + PassInfo *PI = new PassInfo( + "Expand Atomic calls in terms of load-linked & store-conditional", + "atomic-ll-sc", &AtomicExpandLoadLinked::ID, + PassInfo::NormalCtor_t(callDefaultCtor), false, + false, PassInfo::TargetMachineCtor_t( + callTargetMachineCtor)); + Registry.registerPass(*PI, true); + return PI; +} + +void llvm::initializeAtomicExpandLoadLinkedPass(PassRegistry &Registry) { + CALL_ONCE_INITIALIZATION(initializeAtomicExpandLoadLinkedPassOnce) +} + + +FunctionPass *llvm::createAtomicExpandLoadLinkedPass(const TargetMachine *TM) { + return new AtomicExpandLoadLinked(TM); +} + +bool AtomicExpandLoadLinked::runOnFunction(Function &F) { + if (!TLI) + return false; + + SmallVector AtomicInsts; + + // Changing control-flow while iterating through it is a bad idea, so gather a + // list of all atomic instructions before we start. + for (BasicBlock &BB : F) + for (Instruction &Inst : BB) { + if (isa(&Inst) || isa(&Inst) || + (isa(&Inst) && cast(&Inst)->isAtomic()) || + (isa(&Inst) && cast(&Inst)->isAtomic())) + AtomicInsts.push_back(&Inst); + } + + bool MadeChange = false; + for (Instruction *Inst : AtomicInsts) { + if (!TLI->shouldExpandAtomicInIR(Inst)) + continue; + + if (AtomicRMWInst *AI = dyn_cast(Inst)) + MadeChange |= expandAtomicRMW(AI); + else if (AtomicCmpXchgInst *CI = dyn_cast(Inst)) + MadeChange |= expandAtomicCmpXchg(CI); + else if (LoadInst *LI = dyn_cast(Inst)) + MadeChange |= expandAtomicLoad(LI); + else if (StoreInst *SI = dyn_cast(Inst)) + MadeChange |= expandAtomicStore(SI); + else + llvm_unreachable("Unknown atomic instruction"); + } + + return MadeChange; +} + +bool AtomicExpandLoadLinked::expandAtomicLoad(LoadInst *LI) { + // Load instructions don't actually need a leading fence, even in the + // SequentiallyConsistent case. + AtomicOrdering MemOpOrder = + TLI->getInsertFencesForAtomic() ? Monotonic : LI->getOrdering(); + + // The only 64-bit load guaranteed to be single-copy atomic by the ARM ARM is + // an ldrexd (A3.5.3). + IRBuilder<> Builder(LI); + Value *Val = + TLI->emitLoadLinked(Builder, LI->getPointerOperand(), MemOpOrder); + + insertTrailingFence(Builder, LI->getOrdering()); + + LI->replaceAllUsesWith(Val); + LI->eraseFromParent(); + + return true; +} + +bool AtomicExpandLoadLinked::expandAtomicStore(StoreInst *SI) { + // The only atomic 64-bit store on ARM is an strexd that succeeds, which means + // we need a loop and the entire instruction is essentially an "atomicrmw + // xchg" that ignores the value loaded. + IRBuilder<> Builder(SI); + AtomicRMWInst *AI = + Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(), + SI->getValueOperand(), SI->getOrdering()); + SI->eraseFromParent(); + + // Now we have an appropriate swap instruction, lower it as usual. + return expandAtomicRMW(AI); +} + +bool AtomicExpandLoadLinked::expandAtomicRMW(AtomicRMWInst *AI) { + AtomicOrdering Order = AI->getOrdering(); + Value *Addr = AI->getPointerOperand(); + BasicBlock *BB = AI->getParent(); + Function *F = BB->getParent(); + LLVMContext &Ctx = F->getContext(); + + // Given: atomicrmw some_op iN* %addr, iN %incr ordering + // + // The standard expansion we produce is: + // [...] + // fence? + // atomicrmw.start: + // %loaded = @load.linked(%addr) + // %new = some_op iN %loaded, %incr + // %stored = @store_conditional(%new, %addr) + // %try_again = icmp i32 ne %stored, 0 + // br i1 %try_again, label %loop, label %atomicrmw.end + // atomicrmw.end: + // fence? + // [...] + BasicBlock *ExitBB = BB->splitBasicBlock(AI, "atomicrmw.end"); + BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); + + // This grabs the DebugLoc from AI. + IRBuilder<> Builder(AI); + + // The split call above "helpfully" added a branch at the end of BB (to the + // wrong place), but we might want a fence too. It's easiest to just remove + // the branch entirely. + std::prev(BB->end())->eraseFromParent(); + Builder.SetInsertPoint(BB); + AtomicOrdering MemOpOrder = insertLeadingFence(Builder, Order); + Builder.CreateBr(LoopBB); + + // Start the main loop block now that we've taken care of the preliminaries. + Builder.SetInsertPoint(LoopBB); + Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); + + Value *NewVal; + switch (AI->getOperation()) { + case AtomicRMWInst::Xchg: + NewVal = AI->getValOperand(); + break; + case AtomicRMWInst::Add: + NewVal = Builder.CreateAdd(Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::Sub: + NewVal = Builder.CreateSub(Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::And: + NewVal = Builder.CreateAnd(Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::Nand: + NewVal = Builder.CreateAnd(Loaded, Builder.CreateNot(AI->getValOperand()), + "new"); + break; + case AtomicRMWInst::Or: + NewVal = Builder.CreateOr(Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::Xor: + NewVal = Builder.CreateXor(Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::Max: + NewVal = Builder.CreateICmpSGT(Loaded, AI->getValOperand()); + NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::Min: + NewVal = Builder.CreateICmpSLE(Loaded, AI->getValOperand()); + NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::UMax: + NewVal = Builder.CreateICmpUGT(Loaded, AI->getValOperand()); + NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); + break; + case AtomicRMWInst::UMin: + NewVal = Builder.CreateICmpULE(Loaded, AI->getValOperand()); + NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); + break; + default: + llvm_unreachable("Unknown atomic op"); + } + + Value *StoreSuccess = + TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder); + Value *TryAgain = Builder.CreateICmpNE( + StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain"); + Builder.CreateCondBr(TryAgain, LoopBB, ExitBB); + + Builder.SetInsertPoint(ExitBB, ExitBB->begin()); + insertTrailingFence(Builder, Order); + + AI->replaceAllUsesWith(Loaded); + AI->eraseFromParent(); + + return true; +} + +bool AtomicExpandLoadLinked::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { + AtomicOrdering SuccessOrder = CI->getSuccessOrdering(); + AtomicOrdering FailureOrder = CI->getFailureOrdering(); + Value *Addr = CI->getPointerOperand(); + BasicBlock *BB = CI->getParent(); + Function *F = BB->getParent(); + LLVMContext &Ctx = F->getContext(); + + // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord + // + // The full expansion we produce is: + // [...] + // fence? + // cmpxchg.start: + // %loaded = @load.linked(%addr) + // %should_store = icmp eq %loaded, %desired + // br i1 %should_store, label %cmpxchg.trystore, + // label %cmpxchg.end/%cmpxchg.barrier + // cmpxchg.trystore: + // %stored = @store_conditional(%new, %addr) + // %try_again = icmp i32 ne %stored, 0 + // br i1 %try_again, label %loop, label %cmpxchg.end + // cmpxchg.barrier: + // fence? + // br label %cmpxchg.end + // cmpxchg.end: + // [...] + BasicBlock *ExitBB = BB->splitBasicBlock(CI, "cmpxchg.end"); + auto BarrierBB = BasicBlock::Create(Ctx, "cmpxchg.barrier", F, ExitBB); + auto TryStoreBB = BasicBlock::Create(Ctx, "cmpxchg.trystore", F, BarrierBB); + auto LoopBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, TryStoreBB); + + // This grabs the DebugLoc from CI + IRBuilder<> Builder(CI); + + // The split call above "helpfully" added a branch at the end of BB (to the + // wrong place), but we might want a fence too. It's easiest to just remove + // the branch entirely. + std::prev(BB->end())->eraseFromParent(); + Builder.SetInsertPoint(BB); + AtomicOrdering MemOpOrder = insertLeadingFence(Builder, SuccessOrder); + Builder.CreateBr(LoopBB); + + // Start the main loop block now that we've taken care of the preliminaries. + Builder.SetInsertPoint(LoopBB); + Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder); + Value *ShouldStore = + Builder.CreateICmpEQ(Loaded, CI->getCompareOperand(), "should_store"); + + // If the the cmpxchg doesn't actually need any ordering when it fails, we can + // jump straight past that fence instruction (if it exists). + BasicBlock *FailureBB = FailureOrder == Monotonic ? ExitBB : BarrierBB; + Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB); + + Builder.SetInsertPoint(TryStoreBB); + Value *StoreSuccess = TLI->emitStoreConditional( + Builder, CI->getNewValOperand(), Addr, MemOpOrder); + Value *TryAgain = Builder.CreateICmpNE( + StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success"); + Builder.CreateCondBr(TryAgain, LoopBB, BarrierBB); + + // Finally, make sure later instructions don't get reordered with a fence if + // necessary. + Builder.SetInsertPoint(BarrierBB); + insertTrailingFence(Builder, SuccessOrder); + Builder.CreateBr(ExitBB); + + CI->replaceAllUsesWith(Loaded); + CI->eraseFromParent(); + + return true; +} + +AtomicOrdering AtomicExpandLoadLinked::insertLeadingFence(IRBuilder<> &Builder, + AtomicOrdering Ord) { + if (!TLI->getInsertFencesForAtomic()) + return Ord; + + if (Ord == Release || Ord == AcquireRelease || Ord == SequentiallyConsistent) + Builder.CreateFence(Release); + + // The exclusive operations don't need any barrier if we're adding separate + // fences. + return Monotonic; +} + +void AtomicExpandLoadLinked::insertTrailingFence(IRBuilder<> &Builder, + AtomicOrdering Ord) { + if (!TLI->getInsertFencesForAtomic()) + return; + + if (Ord == Acquire || Ord == AcquireRelease) + Builder.CreateFence(Acquire); + else if (Ord == SequentiallyConsistent) + Builder.CreateFence(SequentiallyConsistent); +} Index: lib/CodeGen/CMakeLists.txt =================================================================== --- lib/CodeGen/CMakeLists.txt +++ lib/CodeGen/CMakeLists.txt @@ -2,6 +2,7 @@ AggressiveAntiDepBreaker.cpp AllocationOrder.cpp Analysis.cpp + AtomicExpandLoadLinkedPass.cpp BasicTargetTransformInfo.cpp BranchFolding.cpp CalcSpillWeights.cpp Index: lib/CodeGen/CodeGen.cpp =================================================================== --- lib/CodeGen/CodeGen.cpp +++ lib/CodeGen/CodeGen.cpp @@ -20,6 +20,7 @@ /// initializeCodeGen - Initialize all passes linked into the CodeGen library. void llvm::initializeCodeGen(PassRegistry &Registry) { + initializeAtomicExpandLoadLinkedPass(Registry); initializeBasicTTIPass(Registry); initializeBranchFolderPassPass(Registry); initializeCodeGenPreparePass(Registry); Index: lib/Target/ARM/ARM.h =================================================================== --- lib/Target/ARM/ARM.h +++ lib/Target/ARM/ARM.h @@ -49,8 +49,6 @@ /// \brief Creates an ARM-specific Target Transformation Info pass. ImmutablePass *createARMTargetTransformInfoPass(const ARMBaseTargetMachine *TM); -FunctionPass *createARMAtomicExpandPass(const TargetMachine *TM); - void LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, ARMAsmPrinter &AP); Index: lib/Target/ARM/ARMAtomicExpandPass.cpp =================================================================== --- lib/Target/ARM/ARMAtomicExpandPass.cpp +++ /dev/null @@ -1,406 +0,0 @@ -//===-- ARMAtomicExpandPass.cpp - Expand atomic instructions --------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains a pass (at IR level) to replace atomic instructions with -// appropriate (intrinsic-based) ldrex/strex loops. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "arm-atomic-expand" -#include "ARM.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Debug.h" -#include "llvm/Target/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" -using namespace llvm; - -namespace { - class ARMAtomicExpandPass : public FunctionPass { - const TargetLowering *TLI; - public: - static char ID; // Pass identification, replacement for typeid - explicit ARMAtomicExpandPass(const TargetMachine *TM = 0) - : FunctionPass(ID), TLI(TM->getTargetLowering()) {} - - bool runOnFunction(Function &F) override; - bool expandAtomicInsts(Function &F); - - bool expandAtomicLoad(LoadInst *LI); - bool expandAtomicStore(StoreInst *LI); - bool expandAtomicRMW(AtomicRMWInst *AI); - bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); - - AtomicOrdering insertLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord); - void insertTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord); - - /// Perform a load-linked operation on Addr, returning a "Value *" with the - /// corresponding pointee type. This may entail some non-trivial operations - /// to truncate or reconstruct illegal types since intrinsics must be legal - Value *loadLinked(IRBuilder<> &Builder, Value *Addr, AtomicOrdering Ord); - - /// Perform a store-conditional operation to Addr. Return the status of the - /// store: 0 if the it succeeded, non-zero otherwise. - Value *storeConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, - AtomicOrdering Ord); - - /// Return true if the given (atomic) instruction should be expanded by this - /// pass. - bool shouldExpandAtomic(Instruction *Inst); - }; -} - -char ARMAtomicExpandPass::ID = 0; - -FunctionPass *llvm::createARMAtomicExpandPass(const TargetMachine *TM) { - return new ARMAtomicExpandPass(TM); -} - -bool ARMAtomicExpandPass::runOnFunction(Function &F) { - SmallVector AtomicInsts; - - // Changing control-flow while iterating through it is a bad idea, so gather a - // list of all atomic instructions before we start. - for (BasicBlock &BB : F) - for (Instruction &Inst : BB) { - if (isa(&Inst) || isa(&Inst) || - (isa(&Inst) && cast(&Inst)->isAtomic()) || - (isa(&Inst) && cast(&Inst)->isAtomic())) - AtomicInsts.push_back(&Inst); - } - - bool MadeChange = false; - for (Instruction *Inst : AtomicInsts) { - if (!shouldExpandAtomic(Inst)) - continue; - - if (AtomicRMWInst *AI = dyn_cast(Inst)) - MadeChange |= expandAtomicRMW(AI); - else if (AtomicCmpXchgInst *CI = dyn_cast(Inst)) - MadeChange |= expandAtomicCmpXchg(CI); - else if (LoadInst *LI = dyn_cast(Inst)) - MadeChange |= expandAtomicLoad(LI); - else if (StoreInst *SI = dyn_cast(Inst)) - MadeChange |= expandAtomicStore(SI); - else - llvm_unreachable("Unknown atomic instruction"); - } - - return MadeChange; -} - -bool ARMAtomicExpandPass::expandAtomicLoad(LoadInst *LI) { - // Load instructions don't actually need a leading fence, even in the - // SequentiallyConsistent case. - AtomicOrdering MemOpOrder = - TLI->getInsertFencesForAtomic() ? Monotonic : LI->getOrdering(); - - // The only 64-bit load guaranteed to be single-copy atomic by the ARM ARM is - // an ldrexd (A3.5.3). - IRBuilder<> Builder(LI); - Value *Val = loadLinked(Builder, LI->getPointerOperand(), MemOpOrder); - - insertTrailingFence(Builder, LI->getOrdering()); - - LI->replaceAllUsesWith(Val); - LI->eraseFromParent(); - - return true; -} - -bool ARMAtomicExpandPass::expandAtomicStore(StoreInst *SI) { - // The only atomic 64-bit store on ARM is an strexd that succeeds, which means - // we need a loop and the entire instruction is essentially an "atomicrmw - // xchg" that ignores the value loaded. - IRBuilder<> Builder(SI); - AtomicRMWInst *AI = - Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(), - SI->getValueOperand(), SI->getOrdering()); - SI->eraseFromParent(); - - // Now we have an appropriate swap instruction, lower it as usual. - return expandAtomicRMW(AI); -} - -bool ARMAtomicExpandPass::expandAtomicRMW(AtomicRMWInst *AI) { - AtomicOrdering Order = AI->getOrdering(); - Value *Addr = AI->getPointerOperand(); - BasicBlock *BB = AI->getParent(); - Function *F = BB->getParent(); - LLVMContext &Ctx = F->getContext(); - - // Given: atomicrmw some_op iN* %addr, iN %incr ordering - // - // The standard expansion we produce is: - // [...] - // fence? - // atomicrmw.start: - // %loaded = @load.linked(%addr) - // %new = some_op iN %loaded, %incr - // %stored = @store_conditional(%new, %addr) - // %try_again = icmp i32 ne %stored, 0 - // br i1 %try_again, label %loop, label %atomicrmw.end - // atomicrmw.end: - // fence? - // [...] - BasicBlock *ExitBB = BB->splitBasicBlock(AI, "atomicrmw.end"); - BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); - - // This grabs the DebugLoc from AI. - IRBuilder<> Builder(AI); - - // The split call above "helpfully" added a branch at the end of BB (to the - // wrong place), but we might want a fence too. It's easiest to just remove - // the branch entirely. - std::prev(BB->end())->eraseFromParent(); - Builder.SetInsertPoint(BB); - AtomicOrdering MemOpOrder = insertLeadingFence(Builder, Order); - Builder.CreateBr(LoopBB); - - // Start the main loop block now that we've taken care of the preliminaries. - Builder.SetInsertPoint(LoopBB); - Value *Loaded = loadLinked(Builder, Addr, MemOpOrder); - - Value *NewVal; - switch (AI->getOperation()) { - case AtomicRMWInst::Xchg: - NewVal = AI->getValOperand(); - break; - case AtomicRMWInst::Add: - NewVal = Builder.CreateAdd(Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::Sub: - NewVal = Builder.CreateSub(Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::And: - NewVal = Builder.CreateAnd(Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::Nand: - NewVal = Builder.CreateAnd(Loaded, Builder.CreateNot(AI->getValOperand()), - "new"); - break; - case AtomicRMWInst::Or: - NewVal = Builder.CreateOr(Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::Xor: - NewVal = Builder.CreateXor(Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::Max: - NewVal = Builder.CreateICmpSGT(Loaded, AI->getValOperand()); - NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::Min: - NewVal = Builder.CreateICmpSLE(Loaded, AI->getValOperand()); - NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::UMax: - NewVal = Builder.CreateICmpUGT(Loaded, AI->getValOperand()); - NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); - break; - case AtomicRMWInst::UMin: - NewVal = Builder.CreateICmpULE(Loaded, AI->getValOperand()); - NewVal = Builder.CreateSelect(NewVal, Loaded, AI->getValOperand(), "new"); - break; - default: - llvm_unreachable("Unknown atomic op"); - } - - Value *StoreSuccess = storeConditional(Builder, NewVal, Addr, MemOpOrder); - Value *TryAgain = Builder.CreateICmpNE( - StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain"); - Builder.CreateCondBr(TryAgain, LoopBB, ExitBB); - - Builder.SetInsertPoint(ExitBB, ExitBB->begin()); - insertTrailingFence(Builder, Order); - - AI->replaceAllUsesWith(Loaded); - AI->eraseFromParent(); - - return true; -} - -bool ARMAtomicExpandPass::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { - AtomicOrdering SuccessOrder = CI->getSuccessOrdering(); - AtomicOrdering FailureOrder = CI->getFailureOrdering(); - Value *Addr = CI->getPointerOperand(); - BasicBlock *BB = CI->getParent(); - Function *F = BB->getParent(); - LLVMContext &Ctx = F->getContext(); - - // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord - // - // The full expansion we produce is: - // [...] - // fence? - // cmpxchg.start: - // %loaded = @load.linked(%addr) - // %should_store = icmp eq %loaded, %desired - // br i1 %should_store, label %cmpxchg.trystore, - // label %cmpxchg.end/%cmpxchg.barrier - // cmpxchg.trystore: - // %stored = @store_conditional(%new, %addr) - // %try_again = icmp i32 ne %stored, 0 - // br i1 %try_again, label %loop, label %cmpxchg.end - // cmpxchg.barrier: - // fence? - // br label %cmpxchg.end - // cmpxchg.end: - // [...] - BasicBlock *ExitBB = BB->splitBasicBlock(CI, "cmpxchg.end"); - auto BarrierBB = BasicBlock::Create(Ctx, "cmpxchg.trystore", F, ExitBB); - auto TryStoreBB = BasicBlock::Create(Ctx, "cmpxchg.barrier", F, BarrierBB); - auto LoopBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, TryStoreBB); - - // This grabs the DebugLoc from CI - IRBuilder<> Builder(CI); - - // The split call above "helpfully" added a branch at the end of BB (to the - // wrong place), but we might want a fence too. It's easiest to just remove - // the branch entirely. - std::prev(BB->end())->eraseFromParent(); - Builder.SetInsertPoint(BB); - AtomicOrdering MemOpOrder = insertLeadingFence(Builder, SuccessOrder); - Builder.CreateBr(LoopBB); - - // Start the main loop block now that we've taken care of the preliminaries. - Builder.SetInsertPoint(LoopBB); - Value *Loaded = loadLinked(Builder, Addr, MemOpOrder); - Value *ShouldStore = - Builder.CreateICmpEQ(Loaded, CI->getCompareOperand(), "should_store"); - - // If the the cmpxchg doesn't actually need any ordering when it fails, we can - // jump straight past that fence instruction (if it exists). - BasicBlock *FailureBB = FailureOrder == Monotonic ? ExitBB : BarrierBB; - Builder.CreateCondBr(ShouldStore, TryStoreBB, FailureBB); - - Builder.SetInsertPoint(TryStoreBB); - Value *StoreSuccess = - storeConditional(Builder, CI->getNewValOperand(), Addr, MemOpOrder); - Value *TryAgain = Builder.CreateICmpNE( - StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success"); - Builder.CreateCondBr(TryAgain, LoopBB, BarrierBB); - - // Finally, make sure later instructions don't get reordered with a fence if - // necessary. - Builder.SetInsertPoint(BarrierBB); - insertTrailingFence(Builder, SuccessOrder); - Builder.CreateBr(ExitBB); - - CI->replaceAllUsesWith(Loaded); - CI->eraseFromParent(); - - return true; -} - -Value *ARMAtomicExpandPass::loadLinked(IRBuilder<> &Builder, Value *Addr, - AtomicOrdering Ord) { - Module *M = Builder.GetInsertBlock()->getParent()->getParent(); - Type *ValTy = cast(Addr->getType())->getElementType(); - bool IsAcquire = - Ord == Acquire || Ord == AcquireRelease || Ord == SequentiallyConsistent; - - // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd - // intrinsic must return {i32, i32} and we have to recombine them into a - // single i64 here. - if (ValTy->getPrimitiveSizeInBits() == 64) { - Intrinsic::ID Int = - IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd; - Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int); - - Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); - Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi"); - - Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); - Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); - Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); - Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); - return Builder.CreateOr( - Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64"); - } - - Type *Tys[] = { Addr->getType() }; - Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex; - Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int, Tys); - - return Builder.CreateTruncOrBitCast( - Builder.CreateCall(Ldrex, Addr), - cast(Addr->getType())->getElementType()); -} - -Value *ARMAtomicExpandPass::storeConditional(IRBuilder<> &Builder, Value *Val, - Value *Addr, AtomicOrdering Ord) { - Module *M = Builder.GetInsertBlock()->getParent()->getParent(); - bool IsRelease = - Ord == Release || Ord == AcquireRelease || Ord == SequentiallyConsistent; - - // Since the intrinsics must have legal type, the i64 intrinsics take two - // parameters: "i32, i32". We must marshal Val into the appropriate form - // before the call. - if (Val->getType()->getPrimitiveSizeInBits() == 64) { - Intrinsic::ID Int = - IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd; - Function *Strex = Intrinsic::getDeclaration(M, Int); - Type *Int32Ty = Type::getInt32Ty(M->getContext()); - - Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo"); - Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi"); - Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); - return Builder.CreateCall3(Strex, Lo, Hi, Addr); - } - - Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex; - Type *Tys[] = { Addr->getType() }; - Function *Strex = Intrinsic::getDeclaration(M, Int, Tys); - - return Builder.CreateCall2( - Strex, Builder.CreateZExtOrBitCast( - Val, Strex->getFunctionType()->getParamType(0)), - Addr); -} - -AtomicOrdering ARMAtomicExpandPass::insertLeadingFence(IRBuilder<> &Builder, - AtomicOrdering Ord) { - if (!TLI->getInsertFencesForAtomic()) - return Ord; - - if (Ord == Release || Ord == AcquireRelease || Ord == SequentiallyConsistent) - Builder.CreateFence(Release); - - // The exclusive operations don't need any barrier if we're adding separate - // fences. - return Monotonic; -} - -void ARMAtomicExpandPass::insertTrailingFence(IRBuilder<> &Builder, - AtomicOrdering Ord) { - if (!TLI->getInsertFencesForAtomic()) - return; - - if (Ord == Acquire || Ord == AcquireRelease) - Builder.CreateFence(Acquire); - else if (Ord == SequentiallyConsistent) - Builder.CreateFence(SequentiallyConsistent); -} - -bool ARMAtomicExpandPass::shouldExpandAtomic(Instruction *Inst) { - // Loads and stores less than 64-bits are already atomic; ones above that - // are doomed anyway, so defer to the default libcall and blame the OS when - // things go wrong: - if (StoreInst *SI = dyn_cast(Inst)) - return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() == 64; - else if (LoadInst *LI = dyn_cast(Inst)) - return LI->getType()->getPrimitiveSizeInBits() == 64; - - // For the real atomic operations, we have ldrex/strex up to 64 bits. - return Inst->getType()->getPrimitiveSizeInBits() <= 64; -} Index: lib/Target/ARM/ARMISelLowering.h =================================================================== --- lib/Target/ARM/ARMISelLowering.h +++ lib/Target/ARM/ARMISelLowering.h @@ -384,6 +384,13 @@ bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override; + Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr, + AtomicOrdering Ord) const override; + Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val, + Value *Addr, AtomicOrdering Ord) const override; + + bool shouldExpandAtomicInIR(Instruction *Inst) const override; + protected: std::pair findRepresentativeClass(MVT VT) const override; Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -37,6 +37,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" @@ -10494,3 +10495,83 @@ return false; return true; } + +bool ARMTargetLowering::shouldExpandAtomicInIR(Instruction *Inst) const { + // Loads and stores less than 64-bits are already atomic; ones above that + // are doomed anyway, so defer to the default libcall and blame the OS when + // things go wrong: + if (StoreInst *SI = dyn_cast(Inst)) + return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() == 64; + else if (LoadInst *LI = dyn_cast(Inst)) + return LI->getType()->getPrimitiveSizeInBits() == 64; + + // For the real atomic operations, we have ldrex/strex up to 64 bits. + return Inst->getType()->getPrimitiveSizeInBits() <= 64; +} + +Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr, + AtomicOrdering Ord) const { + Module *M = Builder.GetInsertBlock()->getParent()->getParent(); + Type *ValTy = cast(Addr->getType())->getElementType(); + bool IsAcquire = + Ord == Acquire || Ord == AcquireRelease || Ord == SequentiallyConsistent; + + // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd + // intrinsic must return {i32, i32} and we have to recombine them into a + // single i64 here. + if (ValTy->getPrimitiveSizeInBits() == 64) { + Intrinsic::ID Int = + IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd; + Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int); + + Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); + Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi"); + + Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo"); + Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi"); + Lo = Builder.CreateZExt(Lo, ValTy, "lo64"); + Hi = Builder.CreateZExt(Hi, ValTy, "hi64"); + return Builder.CreateOr( + Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64"); + } + + Type *Tys[] = { Addr->getType() }; + Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex; + Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int, Tys); + + return Builder.CreateTruncOrBitCast( + Builder.CreateCall(Ldrex, Addr), + cast(Addr->getType())->getElementType()); +} + +Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val, + Value *Addr, + AtomicOrdering Ord) const { + Module *M = Builder.GetInsertBlock()->getParent()->getParent(); + bool IsRelease = + Ord == Release || Ord == AcquireRelease || Ord == SequentiallyConsistent; + + // Since the intrinsics must have legal type, the i64 intrinsics take two + // parameters: "i32, i32". We must marshal Val into the appropriate form + // before the call. + if (Val->getType()->getPrimitiveSizeInBits() == 64) { + Intrinsic::ID Int = + IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd; + Function *Strex = Intrinsic::getDeclaration(M, Int); + Type *Int32Ty = Type::getInt32Ty(M->getContext()); + + Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo"); + Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi"); + Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); + return Builder.CreateCall3(Strex, Lo, Hi, Addr); + } + + Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex; + Type *Tys[] = { Addr->getType() }; + Function *Strex = Intrinsic::getDeclaration(M, Int, Tys); + + return Builder.CreateCall2( + Strex, Builder.CreateZExtOrBitCast( + Val, Strex->getFunctionType()->getParamType(0)), + Addr); +} Index: lib/Target/ARM/ARMTargetMachine.cpp =================================================================== --- lib/Target/ARM/ARMTargetMachine.cpp +++ lib/Target/ARM/ARMTargetMachine.cpp @@ -228,7 +228,7 @@ bool ARMPassConfig::addPreISel() { const ARMSubtarget *Subtarget = &getARMSubtarget(); if (Subtarget->hasAnyDataBarrier() && !Subtarget->isThumb1Only()) - addPass(createARMAtomicExpandPass(TM)); + addPass(createAtomicExpandLoadLinkedPass(TM)); if (TM->getOptLevel() != CodeGenOpt::None) addPass(createGlobalMergePass(TM)); Index: lib/Target/ARM/CMakeLists.txt =================================================================== --- lib/Target/ARM/CMakeLists.txt +++ lib/Target/ARM/CMakeLists.txt @@ -17,7 +17,6 @@ add_llvm_target(ARMCodeGen A15SDOptimizer.cpp ARMAsmPrinter.cpp - ARMAtomicExpandPass.cpp ARMBaseInstrInfo.cpp ARMBaseRegisterInfo.cpp ARMCodeEmitter.cpp Index: test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v7.ll =================================================================== --- /dev/null +++ test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v7.ll @@ -0,0 +1,340 @@ +; RUN: opt -S -o - -mtriple=armv7-apple-ios7.0 -atomic-ll-sc %s | FileCheck %s + +define i8 @test_atomic_xchg_i8(i8* %ptr, i8 %xchgend) { +; CHECK-LABEL: @test_atomic_xchg_i8 +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK-NOT: fence +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw xchg i8* %ptr, i8 %xchgend monotonic + ret i8 %res +} + +define i16 @test_atomic_add_i16(i16* %ptr, i16 %addend) { +; CHECK-LABEL: @test_atomic_add_i16 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16 +; CHECK: [[NEWVAL:%.*]] = add i16 [[OLDVAL]], %addend +; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i16 [[OLDVAL]] + %res = atomicrmw add i16* %ptr, i16 %addend seq_cst + ret i16 %res +} + +define i32 @test_atomic_sub_i32(i32* %ptr, i32 %subend) { +; CHECK-LABEL: @test_atomic_sub_i32 +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr) +; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 [[NEWVAL]], i32* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence acquire +; CHECK: ret i32 [[OLDVAL]] + %res = atomicrmw sub i32* %ptr, i32 %subend acquire + ret i32 %res +} + +define i8 @test_atomic_and_i8(i8* %ptr, i8 %andend) { +; CHECK-LABEL: @test_atomic_and_i8 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[NEWVAL:%.*]] = and i8 [[OLDVAL]], %andend +; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK-NOT: fence +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw and i8* %ptr, i8 %andend release + ret i8 %res +} + +define i16 @test_atomic_nand_i16(i16* %ptr, i16 %nandend) { +; CHECK-LABEL: @test_atomic_nand_i16 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16 +; CHECK: [[NEWVAL_TMP:%.*]] = xor i16 %nandend, -1 +; CHECK: [[NEWVAL:%.*]] = and i16 [[OLDVAL]], [[NEWVAL_TMP]] +; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i16 [[OLDVAL]] + %res = atomicrmw nand i16* %ptr, i16 %nandend seq_cst + ret i16 %res +} + +define i64 @test_atomic_or_i64(i64* %ptr, i64 %orend) { +; CHECK-LABEL: @test_atomic_or_i64 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[PTR8]]) +; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0 +; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1 +; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64 +; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64 +; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 +; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] +; CHECK: [[NEWVAL:%.*]] = or i64 [[OLDVAL]], %orend +; CHECK: [[NEWLO:%.*]] = trunc i64 [[NEWVAL]] to i32 +; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 [[NEWVAL]], 32 +; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32 +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]]) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i64 [[OLDVAL]] + %res = atomicrmw or i64* %ptr, i64 %orend seq_cst + ret i64 %res +} + +define i8 @test_atomic_xor_i8(i8* %ptr, i8 %xorend) { +; CHECK-LABEL: @test_atomic_xor_i8 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[NEWVAL:%.*]] = xor i8 [[OLDVAL]], %xorend +; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw xor i8* %ptr, i8 %xorend seq_cst + ret i8 %res +} + +define i8 @test_atomic_max_i8(i8* %ptr, i8 %maxend) { +; CHECK-LABEL: @test_atomic_max_i8 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[WANT_OLD:%.*]] = icmp sgt i8 [[OLDVAL]], %maxend +; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %maxend +; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw max i8* %ptr, i8 %maxend seq_cst + ret i8 %res +} + +define i8 @test_atomic_min_i8(i8* %ptr, i8 %minend) { +; CHECK-LABEL: @test_atomic_min_i8 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[WANT_OLD:%.*]] = icmp sle i8 [[OLDVAL]], %minend +; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %minend +; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw min i8* %ptr, i8 %minend seq_cst + ret i8 %res +} + +define i8 @test_atomic_umax_i8(i8* %ptr, i8 %umaxend) { +; CHECK-LABEL: @test_atomic_umax_i8 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[WANT_OLD:%.*]] = icmp ugt i8 [[OLDVAL]], %umaxend +; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %umaxend +; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw umax i8* %ptr, i8 %umaxend seq_cst + ret i8 %res +} + +define i8 @test_atomic_umin_i8(i8* %ptr, i8 %uminend) { +; CHECK-LABEL: @test_atomic_umin_i8 +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[WANT_OLD:%.*]] = icmp ule i8 [[OLDVAL]], %uminend +; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %uminend +; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK: fence seq_cst +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw umin i8* %ptr, i8 %uminend seq_cst + ret i8 %res +} + +define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) { +; CHECK-LABEL: @test_cmpxchg_i8_seqcst_seqcst +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[BARRIER:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK: fence seq_cst +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i8 [[OLDVAL]] + + %old = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst + ret i8 %old +} + +define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newval) { +; CHECK-LABEL: @test_cmpxchg_i16_seqcst_monotonic +; CHECK: fence release +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[DONE:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK: fence seq_cst +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i16 [[OLDVAL]] + + %old = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic + ret i16 %old +} + +define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newval) { +; CHECK-LABEL: @test_cmpxchg_i32_acquire_acquire +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr) +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[DONE:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK: fence acquire +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i32 [[OLDVAL]] + + %old = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire + ret i32 %old +} + +define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %newval) { +; CHECK-LABEL: @test_cmpxchg_i64_monotonic_monotonic +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[PTR8]]) +; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0 +; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1 +; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64 +; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64 +; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 +; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[DONE:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 +; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32 +; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32 +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]]) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK-NOT: fence +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i64 [[OLDVAL]] + + %old = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic + ret i64 %old +} \ No newline at end of file Index: test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v8.ll =================================================================== --- /dev/null +++ test/Transforms/AtomicExpandLoadLinked/ARM/atomic-expansion-v8.ll @@ -0,0 +1,202 @@ +; RUN: opt -S -o - -mtriple=armv8-linux-gnueabihf -atomic-ll-sc %s | FileCheck %s + +define i8 @test_atomic_xchg_i8(i8* %ptr, i8 %xchgend) { +; CHECK-LABEL: @test_atomic_xchg_i8 +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 +; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK-NOT: fence +; CHECK: ret i8 [[OLDVAL]] + %res = atomicrmw xchg i8* %ptr, i8 %xchgend monotonic + ret i8 %res +} + +define i16 @test_atomic_add_i16(i16* %ptr, i16 %addend) { +; CHECK-LABEL: @test_atomic_add_i16 +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16 +; CHECK: [[NEWVAL:%.*]] = add i16 [[OLDVAL]], %addend +; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i16(i32 [[NEWVAL32]], i16* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK-NOT: fence +; CHECK: ret i16 [[OLDVAL]] + %res = atomicrmw add i16* %ptr, i16 %addend seq_cst + ret i16 %res +} + +define i32 @test_atomic_sub_i32(i32* %ptr, i32 %subend) { +; CHECK-LABEL: @test_atomic_sub_i32 +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr) +; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 [[NEWVAL]], i32* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK-NOT: fence +; CHECK: ret i32 [[OLDVAL]] + %res = atomicrmw sub i32* %ptr, i32 %subend acquire + ret i32 %res +} + +define i64 @test_atomic_or_i64(i64* %ptr, i64 %orend) { +; CHECK-LABEL: @test_atomic_or_i64 +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldaexd(i8* [[PTR8]]) +; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0 +; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1 +; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64 +; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64 +; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 +; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] +; CHECK: [[NEWVAL:%.*]] = or i64 [[OLDVAL]], %orend +; CHECK: [[NEWLO:%.*]] = trunc i64 [[NEWVAL]] to i32 +; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 [[NEWVAL]], 32 +; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32 +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]]) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] +; CHECK: [[END]]: +; CHECK-NOT: fence +; CHECK: ret i64 [[OLDVAL]] + %res = atomicrmw or i64* %ptr, i64 %orend seq_cst + ret i64 %res +} + +define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) { +; CHECK-LABEL: @test_cmpxchg_i8_seqcst_seqcst +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i8(i8* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[BARRIER:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i8(i32 [[NEWVAL32]], i8* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK-NOT: fence +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i8 [[OLDVAL]] + + %old = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst + ret i8 %old +} + +define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newval) { +; CHECK-LABEL: @test_cmpxchg_i16_seqcst_monotonic +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr) +; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[DONE:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i16(i32 [[NEWVAL32]], i16* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK-NOT: fence +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i16 [[OLDVAL]] + + %old = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic + ret i16 %old +} + +define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newval) { +; CHECK-LABEL: @test_cmpxchg_i32_acquire_acquire +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr) +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[DONE:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK-NOT: fence +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i32 [[OLDVAL]] + + %old = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire + ret i32 %old +} + +define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %newval) { +; CHECK-LABEL: @test_cmpxchg_i64_monotonic_monotonic +; CHECK-NOT: fence +; CHECK: br label %[[LOOP:.*]] + +; CHECK: [[LOOP]]: +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[PTR8]]) +; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0 +; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1 +; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64 +; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64 +; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 +; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] +; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired +; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[DONE:.*]] + +; CHECK: [[TRY_STORE]]: +; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 +; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32 +; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32 +; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8* +; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]]) +; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 +; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]] + +; CHECK: [[BARRIER]]: +; CHECK-NOT: fence +; CHECK: br label %[[DONE:.*]] + +; CHECK: [[DONE]]: +; CHECK: ret i64 [[OLDVAL]] + + %old = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic + ret i64 %old +} \ No newline at end of file Index: test/Transforms/AtomicExpandLoadLinked/ARM/lit.local.cfg =================================================================== --- /dev/null +++ test/Transforms/AtomicExpandLoadLinked/ARM/lit.local.cfg @@ -0,0 +1,4 @@ +targets = set(config.root.targets_to_build.split()) +if not 'ARM' in targets: + config.unsupported = True + Index: tools/opt/opt.cpp =================================================================== --- tools/opt/opt.cpp +++ tools/opt/opt.cpp @@ -341,8 +341,9 @@ initializeInstrumentation(Registry); initializeTarget(Registry); // For codegen passes, only passes that do IR to IR transformation are - // supported. For now, just add CodeGenPrepare. + // supported. initializeCodeGenPreparePass(Registry); + initializeAtomicExpandLoadLinkedPass(Registry); #ifdef LINK_POLLY_INTO_TOOLS polly::initializePollyPasses(Registry);