Index: include/llvm/CodeGen/Passes.h =================================================================== --- include/llvm/CodeGen/Passes.h +++ include/llvm/CodeGen/Passes.h @@ -28,12 +28,14 @@ class TargetMachine; class TargetRegisterClass; class raw_ostream; +struct SimplifyCFGOptions; } // End llvm namespace /// List of target independent CodeGen pass IDs. namespace llvm { FunctionPass *createAtomicExpandPass(); + FunctionPass *createAtomicExpandPass(const SimplifyCFGOptions &Opts); /// createUnreachableBlockEliminationPass - The LLVM code generator does not /// work well with unreachable basic blocks (what live ranges make sense for a Index: lib/CodeGen/AtomicExpandPass.cpp =================================================================== --- lib/CodeGen/AtomicExpandPass.cpp +++ lib/CodeGen/AtomicExpandPass.cpp @@ -18,6 +18,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/Utils/Local.h" #include "llvm/CodeGen/AtomicExpandUtils.h" #include "llvm/CodeGen/RuntimeLibcalls.h" #include "llvm/CodeGen/TargetLowering.h" @@ -58,16 +60,26 @@ class AtomicExpand: public FunctionPass { const TargetLowering *TLI = nullptr; + const TargetTransformInfo *TTI = nullptr; + // If the CFG simplification options are present, the newly created + // basic blocks will be simplified at the end of runOnFunction. + Optional SimplifyOpts; + SmallVector SimplifyBlocks; public: static char ID; // Pass identification, replacement for typeid - AtomicExpand() : FunctionPass(ID) { + AtomicExpand(Optional Opts = None) + : FunctionPass(ID), SimplifyOpts(Opts) { initializeAtomicExpandPass(*PassRegistry::getPassRegistry()); } bool runOnFunction(Function &F) override; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + } + private: bool bracketInstWithFences(Instruction *I, AtomicOrdering Order); IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL); @@ -125,7 +137,13 @@ INITIALIZE_PASS(AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", false, false) -FunctionPass *llvm::createAtomicExpandPass() { return new AtomicExpand(); } +FunctionPass *llvm::createAtomicExpandPass() { + return new AtomicExpand(None); +} + +FunctionPass *llvm::createAtomicExpandPass(const SimplifyCFGOptions &Opts) { + return new AtomicExpand(Opts); +} // Helper functions to retrieve the size of atomic instructions. static unsigned getAtomicOpSize(LoadInst *LI) { @@ -200,6 +218,8 @@ if (!TM.getSubtargetImpl(F)->enableAtomicExpand()) return false; TLI = TM.getSubtargetImpl(F)->getTargetLowering(); + TTI = &getAnalysis().getTTI(F); + SimplifyBlocks.clear(); SmallVector AtomicInsts; @@ -334,6 +354,11 @@ } } } + + if (SimplifyOpts) + for (BasicBlock *B : SimplifyBlocks) + simplifyCFG(B, *TTI, *SimplifyOpts); + return MadeChange; } @@ -857,6 +882,8 @@ CI->replaceAllUsesWith(Res); CI->eraseFromParent(); + + SimplifyBlocks.insert(SimplifyBlocks.end(), { EndBB, FailureBB, LoopBB }); } void AtomicExpand::expandAtomicOpToLLSC( @@ -913,6 +940,8 @@ Builder.CreateCondBr(TryAgain, LoopBB, ExitBB); Builder.SetInsertPoint(ExitBB, ExitBB->begin()); + + SimplifyBlocks.insert(SimplifyBlocks.end(), { ExitBB, LoopBB }); return Loaded; } @@ -1181,6 +1210,11 @@ } CI->eraseFromParent(); + + SimplifyBlocks.insert(SimplifyBlocks.end(), + { ExitBB, FailureBB, NoStoreBB, SuccessBB, + ReleasedLoadBB, TryStoreBB, ReleasingStoreBB, + StartBB }); return true; } Index: lib/Target/Hexagon/HexagonTargetMachine.cpp =================================================================== --- lib/Target/Hexagon/HexagonTargetMachine.cpp +++ lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -17,6 +17,7 @@ #include "HexagonMachineScheduler.h" #include "HexagonTargetObjectFile.h" #include "HexagonTargetTransformInfo.h" +#include "llvm/Analysis/Utils/Local.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/LegacyPassManager.h" @@ -308,7 +309,7 @@ addPass(createDeadCodeEliminationPass()); } - addPass(createAtomicExpandPass()); + addPass(createAtomicExpandPass(SimplifyCFGOptions())); if (!NoOpt) { if (EnableLoopPrefetch) addPass(createLoopDataPrefetchPass()); Index: test/CodeGen/Hexagon/intrinsics/atomicrmw_addsub_native.ll =================================================================== --- test/CodeGen/Hexagon/intrinsics/atomicrmw_addsub_native.ll +++ test/CodeGen/Hexagon/intrinsics/atomicrmw_addsub_native.ll @@ -39,7 +39,7 @@ ; CHECK: [[RESULT_REG:r[0-9]+]] = [[BINARY_OP]]([[LOCKED_READ_REG]],[[FIRST_VALUE]]) ; CHECK: memw_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memw(gp+#i32Result) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 @@ -60,7 +60,7 @@ ; CHECK: [[RESULT_REG:r[0-9]+:[0-9]+]] = [[BINARY_OP]]([[LOCKED_READ_REG]],[[FIRST_VALUE]]) ; CHECK: memd_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memd(gp+#i64Result) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 @@ -81,7 +81,7 @@ ; CHECK: [[RESULT_REG:r[0-9]+]] = [[BINARY_OP]]([[LOCKED_READ_REG]],[[FIRST_VALUE]]) ; CHECK: memw_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memw(gp+#ptrResult) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 Index: test/CodeGen/Hexagon/intrinsics/atomicrmw_bitwise_native.ll =================================================================== --- test/CodeGen/Hexagon/intrinsics/atomicrmw_bitwise_native.ll +++ test/CodeGen/Hexagon/intrinsics/atomicrmw_bitwise_native.ll @@ -32,7 +32,7 @@ ; CHECK: [[RESULT_REG:r[0-9]+]] = [[BINARY_OP]]([[LOCKED_READ_REG]],[[FIRST_VALUE]]) ; CHECK: memw_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memw(gp+#g2) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 define void @f0() { @@ -53,7 +53,7 @@ ; CHECK: [[RESULT_REG:r[:0-9]+]] = [[BINARY_OP]]([[LOCKED_READ_REG]],[[FIRST_VALUE]]) ; CHECK: memd_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memd(gp+#g5) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 define void @f1() { Index: test/CodeGen/Hexagon/intrinsics/atomicrmw_nand.ll =================================================================== --- test/CodeGen/Hexagon/intrinsics/atomicrmw_nand.ll +++ test/CodeGen/Hexagon/intrinsics/atomicrmw_nand.ll @@ -22,7 +22,7 @@ ; CHECK: [[RESULT_REG:r[0-9]+]] = sub(#-1,[[AND_RESULT_REG]]) ; CHECK: memw_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memw(gp+#g2) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 define void @f0() { @@ -44,7 +44,7 @@ ; CHECK: [[RESULT_REG:r[:0-9]+]] = not([[AND_RESULT_REG]]) ; CHECK: memd_locked([[SECOND_ADDR]],[[LOCK_PRED_REG:p[0-9]+]]) = [[RESULT_REG]] -; CHECK: cmp.eq{{.*}}jump{{.*}}[[FAIL_LABEL]] +; CHECK: if (![[LOCK_PRED_REG]]) jump{{.*}}[[FAIL_LABEL]] ; CHECK-DAG: memd(gp+#g5) = [[LOCKED_READ_REG]] ; CHECK-DAG: jumpr r31 define void @f1() {