diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -105,8 +105,8 @@ // instead of the cross product using AA to identify aliasing of the memory // location we are interested in. static cl::opt -LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0), - cl::desc("How many instruction to cross product using AA")); + LICMN2Threshold("licm-n2-threshold", cl::Hidden, cl::init(0), + cl::desc("How many instruction to cross product using AA")); // Experimental option to allow imprecision in LICM in pathological cases, in // exchange for faster compile. This is to be removed if MemorySSA starts to @@ -149,9 +149,9 @@ const LoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE, const Instruction *CtxI = nullptr); -static bool pointerInvalidatedByLoop(MemoryLocation MemLoc, - AliasSetTracker *CurAST, Loop *CurLoop, - AliasAnalysis *AA); +static bool pointerAccessedByLoop(MemoryLocation MemLoc, + AliasSetTracker *CurAST, Loop *CurLoop, + AliasAnalysis *AA, bool Invalidate); static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU, Loop *CurLoop, SinkAndHoistLICMFlags &Flags); @@ -1115,8 +1115,8 @@ bool Invalidated; if (CurAST) - Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST, - CurLoop, AA); + Invalidated = pointerAccessedByLoop(MemoryLocation::get(LI), CurAST, + CurLoop, AA, true); else Invalidated = pointerInvalidatedByLoopWithMSSA( MSSA, cast(MSSA->getMemoryAccess(LI)), CurLoop, *Flags); @@ -1151,11 +1151,28 @@ // Handle simple cases by querying alias analysis. FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI); - if (Behavior == FMRB_DoesNotAccessMemory) + if (AliasAnalysis::doesNotReadMemory(Behavior)) { + // A readnone/writeonly function can be moved if it does not access + // arguments, or it does and the arguments do not alias in the loop. + if (!AliasAnalysis::onlyAccessesInaccessibleOrArgMem(Behavior)) + return false; + if (AliasAnalysis::doesAccessArgPointees(Behavior)) { + for (Value *Op : CI->arg_operands()) { + if (CurAST && + pointerAccessedByLoop( + MemoryLocation(Op, LocationSize::unknown(), AAMDNodes()), + CurAST, CurLoop, AA, false)) + return false; + else if (!isNoAliasArgument(Op) && !isNoAliasCall(Op)) + return false; + } + } + return true; + } if (AliasAnalysis::onlyReadsMemory(Behavior)) { // A readonly argmemonly function only reads from memory pointed to by - // it's arguments with arbitrary offsets. If we can prove there are no + // its arguments with arbitrary offsets. If we can prove there are no // writes to this memory in the loop, we can hoist or sink. if (AliasAnalysis::onlyAccessesArgPointees(Behavior)) { // TODO: expand to writeable arguments @@ -1163,9 +1180,9 @@ if (Op->getType()->isPointerTy()) { bool Invalidated; if (CurAST) - Invalidated = pointerInvalidatedByLoop( + Invalidated = pointerAccessedByLoop( MemoryLocation(Op, LocationSize::unknown(), AAMDNodes()), - CurAST, CurLoop, AA); + CurAST, CurLoop, AA, true); else Invalidated = pointerInvalidatedByLoopWithMSSA( MSSA, cast(MSSA->getMemoryAccess(CI)), CurLoop, @@ -1993,7 +2010,7 @@ // Note that proving a load safe to speculate requires proving // sufficient alignment at the target location. Proving it guaranteed // to execute does as well. Thus we can increase our guaranteed - // alignment as well. + // alignment as well. if (!DereferenceableInPH || (InstAlignment > Alignment)) if (isSafeToExecuteUnconditionally(*Load, DT, CurLoop, SafetyInfo, ORE, Preheader->getTerminator())) { @@ -2247,14 +2264,15 @@ LICM.getLoopToAliasSetMap().erase(L); } -static bool pointerInvalidatedByLoop(MemoryLocation MemLoc, - AliasSetTracker *CurAST, Loop *CurLoop, - AliasAnalysis *AA) { - // First check to see if any of the basic blocks in CurLoop invalidate *V. - bool isInvalidatedAccordingToAST = CurAST->getAliasSetFor(MemLoc).isMod(); +static bool pointerAccessedByLoop(MemoryLocation MemLoc, + AliasSetTracker *CurAST, Loop *CurLoop, + AliasAnalysis *AA, bool Invalidate) { + // First check to see if any of the basic blocks in CurLoop access *V. + auto &AS = CurAST->getAliasSetFor(MemLoc); + auto isAccesssedAccordingToAST = Invalidate ? AS.isMod() : AS.isRef(); - if (!isInvalidatedAccordingToAST || !LICMN2Theshold) - return isInvalidatedAccordingToAST; + if (!isAccesssedAccordingToAST || !LICMN2Threshold) + return isAccesssedAccordingToAST; // Check with a diagnostic analysis if we can refine the information above. // This is to identify the limitations of using the AST. @@ -2277,14 +2295,14 @@ int N = 0; for (BasicBlock *BB : CurLoop->getBlocks()) for (Instruction &I : *BB) { - if (N >= LICMN2Theshold) { + if (N >= LICMN2Threshold) { LLVM_DEBUG(dbgs() << "Alasing N2 threshold exhausted for " << *(MemLoc.Ptr) << "\n"); return true; } N++; auto Res = AA->getModRefInfo(&I, MemLoc); - if (isModSet(Res)) { + if (Invalidate ? isModSet(Res) : isRefSet(Res)) { LLVM_DEBUG(dbgs() << "Aliasing failed on " << I << " for " << *(MemLoc.Ptr) << "\n"); return true; diff --git a/llvm/test/Transforms/LICM/hoist-attributes.ll b/llvm/test/Transforms/LICM/hoist-attributes.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LICM/hoist-attributes.ll @@ -0,0 +1,101 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S < %s -licm | FileCheck %s + +declare i32 @foo1a() readnone nounwind +declare i32 @foo1b(i8* readnone) readnone nounwind +declare i32 @foo1c(i8* readnone) nounwind + +declare void @foo2a() inaccessiblememonly writeonly nounwind +declare void @foo2b(i8*) inaccessiblemem_or_argmemonly writeonly nounwind +declare void @foo2c(i8* readnone) inaccessiblememonly writeonly nounwind +declare void @foo2d(i8* writeonly) nounwind + +@str = internal global [4 x i8] c"foo\00", align 1 + +declare noalias i8* @malloc() + +; Functions that are nounwind and readnone (foo1a, foo1b) can be hoisted +define i32 @test1(i1 %c) { +; CHECK-LABEL: @test1( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[STR:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* @str, i64 0, i64 0 +; CHECK-NEXT: [[RET1A:%.*]] = call i32 @foo1a() +; CHECK-NEXT: [[RET1B:%.*]] = call i32 @foo1b(i8* [[STR]]) +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD1C:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[ADD1A:%.*]] = add nuw i32 [[SUM]], [[RET1A]] +; CHECK-NEXT: [[ADD1B:%.*]] = add nuw i32 [[ADD1A]], [[RET1B]] +; CHECK-NEXT: [[RET1C:%.*]] = call i32 @foo1c(i8* [[STR]]) +; CHECK-NEXT: [[ADD1C]] = add nuw i32 [[ADD1B]], [[RET1C]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP]], label [[OUT:%.*]] +; CHECK: out: +; CHECK-NEXT: [[ADD1C_LCSSA:%.*]] = phi i32 [ [[ADD1C]], [[LOOP]] ] +; CHECK-NEXT: ret i32 [[ADD1C_LCSSA]] +; +entry: + %str = getelementptr inbounds [4 x i8], [4 x i8]* @str, i64 0, i64 0 + br label %loop +loop: + %sum = phi i32 [ 0, %entry ], [ %add1c, %loop ] + %ret1a = call i32 @foo1a() + %add1a = add nuw i32 %sum, %ret1a + %ret1b = call i32 @foo1b(i8* %str) + %add1b = add nuw i32 %add1a, %ret1b + %ret1c = call i32 @foo1c(i8* %str) + %add1c = add nuw i32 %add1b, %ret1c + br i1 %c, label %loop, label %out +out: + ret i32 %add1c +} + +; Only writes to inaccessible or argument memory (foo2a, foo2b, foo2c), can all +; be hoisted since arguments don't alias or aren't accessed +define void @test2abc(i1 %c) { +; CHECK-LABEL: @test2abc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MEM:%.*]] = call noalias i8* @malloc() +; CHECK-NEXT: [[STR:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* @str, i64 0, i64 0 +; CHECK-NEXT: call void @foo2a() +; CHECK-NEXT: call void @foo2b(i8* [[MEM]]) +; CHECK-NEXT: call void @foo2c(i8* [[STR]]) +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP]], label [[OUT:%.*]] +; CHECK: out: +; CHECK-NEXT: ret void +; +entry: + %mem = call noalias i8* @malloc() + %str = getelementptr inbounds [4 x i8], [4 x i8]* @str, i64 0, i64 0 + br label %loop +loop: + call void @foo2a() + call void @foo2b(i8* %mem) + call void @foo2c(i8* %str) + br i1 %c, label %loop, label %out +out: + ret void +} + +; May read/write memory that could alias (foo2d), can't be hoisted +define void @test2d(i1 %c) { +; CHECK-LABEL: @test2d( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[STR:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* @str, i64 0, i64 0 +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: call void @foo2d(i8* [[STR]]) +; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP]], label [[OUT:%.*]] +; CHECK: out: +; CHECK-NEXT: ret void +; +entry: + %str = getelementptr inbounds [4 x i8], [4 x i8]* @str, i64 0, i64 0 + br label %loop +loop: + call void @foo2d(i8* %str) + br i1 %c, label %loop, label %out +out: + ret void +}