Index: llvm/include/llvm/Transforms/Scalar/LowerAtomicPass.h =================================================================== --- /dev/null +++ llvm/include/llvm/Transforms/Scalar/LowerAtomicPass.h @@ -0,0 +1,30 @@ +//===- LowerAtomicPass.h - Lower atomic intrinsics --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +// This pass lowers atomic intrinsics to non-atomic form for use in a known +// non-preemptible environment. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERATOMICPASS_H +#define LLVM_TRANSFORMS_SCALAR_LOWERATOMICPASS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// A pass that lowers atomic intrinsic into non-atomic intrinsics. +class LowerAtomicPass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &); + static bool isRequired() { return true; } +}; + +} + +#endif // LLVM_TRANSFORMS_SCALAR_LOWERATOMICPASS_H Index: llvm/include/llvm/Transforms/Utils/LowerAtomic.h =================================================================== --- llvm/include/llvm/Transforms/Utils/LowerAtomic.h +++ llvm/include/llvm/Transforms/Utils/LowerAtomic.h @@ -1,4 +1,4 @@ -//===- LowerAtomic.cpp - Lower atomic intrinsics ----------------*- C++ -*-===// +//===- LowerAtomic.h - Lower atomic intrinsics ------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -14,18 +14,13 @@ #ifndef LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H #define LLVM_TRANSFORMS_SCALAR_LOWERATOMIC_H -#include "llvm/IR/PassManager.h" - namespace llvm { +class AtomicCmpXchgInst; +class AtomicRMWInst; -/// A pass that lowers atomic intrinsic into non-atomic intrinsics. -class LowerAtomicPass : public PassInfoMixin { -public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &); - static bool isRequired() { return true; } -}; +/// Convert the given Cmpxchg into primitive load and compare. +bool lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI); -class AtomicRMWInst; /// Convert the given RMWI into primitive load and stores, /// assuming that doing so is legal. Return true if the lowering /// succeeds. Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -185,7 +185,7 @@ #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h" #include "llvm/Transforms/Scalar/LoopUnrollPass.h" #include "llvm/Transforms/Scalar/LoopVersioningLICM.h" -#include "llvm/Transforms/Scalar/LowerAtomic.h" +#include "llvm/Transforms/Scalar/LowerAtomicPass.h" #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h" Index: llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp =================================================================== --- llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp +++ llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp @@ -17,7 +17,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" -#include "llvm/Transforms/Scalar/LowerAtomic.h" +#include "llvm/Transforms/Utils/LowerAtomic.h" #include "MCTargetDesc/NVPTXBaseInfo.h" using namespace llvm; Index: llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp =================================================================== --- llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -30,7 +30,7 @@ #include "llvm/MC/TargetRegistry.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Scalar.h" -#include "llvm/Transforms/Scalar/LowerAtomic.h" +#include "llvm/Transforms/Scalar/LowerAtomicPass.h" #include "llvm/Transforms/Utils.h" using namespace llvm; Index: llvm/lib/Transforms/Scalar/CMakeLists.txt =================================================================== --- llvm/lib/Transforms/Scalar/CMakeLists.txt +++ llvm/lib/Transforms/Scalar/CMakeLists.txt @@ -47,7 +47,7 @@ LoopUnrollAndJamPass.cpp LoopUnswitch.cpp LoopVersioningLICM.cpp - LowerAtomic.cpp + LowerAtomicPass.cpp LowerConstantIntrinsics.cpp LowerExpectIntrinsic.cpp LowerGuardIntrinsic.cpp Index: llvm/lib/Transforms/Scalar/LowerAtomicPass.cpp =================================================================== --- /dev/null +++ llvm/lib/Transforms/Scalar/LowerAtomicPass.cpp @@ -0,0 +1,99 @@ +//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers atomic intrinsics to non-atomic form for use in a known +// non-preemptible environment. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Scalar/LowerAtomicPass.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Utils/LowerAtomic.h" +using namespace llvm; + +#define DEBUG_TYPE "loweratomic" + +static bool LowerFenceInst(FenceInst *FI) { + FI->eraseFromParent(); + return true; +} + +static bool LowerLoadInst(LoadInst *LI) { + LI->setAtomic(AtomicOrdering::NotAtomic); + return true; +} + +static bool LowerStoreInst(StoreInst *SI) { + SI->setAtomic(AtomicOrdering::NotAtomic); + return true; +} + +static bool runOnBasicBlock(BasicBlock &BB) { + bool Changed = false; + for (Instruction &Inst : make_early_inc_range(BB)) { + if (FenceInst *FI = dyn_cast(&Inst)) + Changed |= LowerFenceInst(FI); + else if (AtomicCmpXchgInst *CXI = dyn_cast(&Inst)) + Changed |= lowerAtomicCmpXchgInst(CXI); + else if (AtomicRMWInst *RMWI = dyn_cast(&Inst)) + Changed |= lowerAtomicRMWInst(RMWI); + else if (LoadInst *LI = dyn_cast(&Inst)) { + if (LI->isAtomic()) + LowerLoadInst(LI); + } else if (StoreInst *SI = dyn_cast(&Inst)) { + if (SI->isAtomic()) + LowerStoreInst(SI); + } + } + return Changed; +} + +static bool lowerAtomics(Function &F) { + bool Changed = false; + for (BasicBlock &BB : F) { + Changed |= runOnBasicBlock(BB); + } + return Changed; +} + +PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) { + if (lowerAtomics(F)) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} + +namespace { +class LowerAtomicLegacyPass : public FunctionPass { +public: + static char ID; + + LowerAtomicLegacyPass() : FunctionPass(ID) { + initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override { + // Don't skip optnone functions; atomics still need to be lowered. + FunctionAnalysisManager DummyFAM; + auto PA = Impl.run(F, DummyFAM); + return !PA.areAllPreserved(); + } + +private: + LowerAtomicPass Impl; + }; +} + +char LowerAtomicLegacyPass::ID = 0; +INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic", + "Lower atomic intrinsics to non-atomic form", false, false) + +Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); } Index: llvm/lib/Transforms/Utils/CMakeLists.txt =================================================================== --- llvm/lib/Transforms/Utils/CMakeLists.txt +++ llvm/lib/Transforms/Utils/CMakeLists.txt @@ -44,6 +44,7 @@ LoopUnrollRuntime.cpp LoopUtils.cpp LoopVersioning.cpp + LowerAtomic.cpp LowerGlobalDtors.cpp LowerInvoke.cpp LowerMemIntrinsics.cpp Index: llvm/lib/Transforms/Utils/LowerAtomic.cpp =================================================================== --- llvm/lib/Transforms/Utils/LowerAtomic.cpp +++ llvm/lib/Transforms/Utils/LowerAtomic.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/Scalar/LowerAtomic.h" +#include "llvm/Transforms/Utils/LowerAtomic.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/InitializePasses.h" @@ -21,7 +21,7 @@ #define DEBUG_TYPE "loweratomic" -static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { +bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { IRBuilder<> Builder(CXI); Value *Ptr = CXI->getPointerOperand(); Value *Cmp = CXI->getCompareOperand(); @@ -99,79 +99,3 @@ RMWI->eraseFromParent(); return true; } - -static bool LowerFenceInst(FenceInst *FI) { - FI->eraseFromParent(); - return true; -} - -static bool LowerLoadInst(LoadInst *LI) { - LI->setAtomic(AtomicOrdering::NotAtomic); - return true; -} - -static bool LowerStoreInst(StoreInst *SI) { - SI->setAtomic(AtomicOrdering::NotAtomic); - return true; -} - -static bool runOnBasicBlock(BasicBlock &BB) { - bool Changed = false; - for (Instruction &Inst : make_early_inc_range(BB)) { - if (FenceInst *FI = dyn_cast(&Inst)) - Changed |= LowerFenceInst(FI); - else if (AtomicCmpXchgInst *CXI = dyn_cast(&Inst)) - Changed |= LowerAtomicCmpXchgInst(CXI); - else if (AtomicRMWInst *RMWI = dyn_cast(&Inst)) - Changed |= lowerAtomicRMWInst(RMWI); - else if (LoadInst *LI = dyn_cast(&Inst)) { - if (LI->isAtomic()) - LowerLoadInst(LI); - } else if (StoreInst *SI = dyn_cast(&Inst)) { - if (SI->isAtomic()) - LowerStoreInst(SI); - } - } - return Changed; -} - -static bool lowerAtomics(Function &F) { - bool Changed = false; - for (BasicBlock &BB : F) { - Changed |= runOnBasicBlock(BB); - } - return Changed; -} - -PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) { - if (lowerAtomics(F)) - return PreservedAnalyses::none(); - return PreservedAnalyses::all(); -} - -namespace { -class LowerAtomicLegacyPass : public FunctionPass { -public: - static char ID; - - LowerAtomicLegacyPass() : FunctionPass(ID) { - initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override { - // Don't skip optnone functions; atomics still need to be lowered. - FunctionAnalysisManager DummyFAM; - auto PA = Impl.run(F, DummyFAM); - return !PA.areAllPreserved(); - } - -private: - LowerAtomicPass Impl; - }; -} - -char LowerAtomicLegacyPass::ID = 0; -INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic", - "Lower atomic intrinsics to non-atomic form", false, false) - -Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); }