diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -177,21 +177,11 @@ /// Register the location (instructions are given increasing numbers) /// of a write access. - void addAccess(StoreInst *SI) { - Value *Ptr = SI->getPointerOperand(); - Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx); - InstMap.push_back(SI); - ++AccessIdx; - } + void addAccess(StoreInst *SI); /// Register the location (instructions are given increasing numbers) /// of a write access. - void addAccess(LoadInst *LI) { - Value *Ptr = LI->getPointerOperand(); - Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx); - InstMap.push_back(LI); - ++AccessIdx; - } + void addAccess(LoadInst *LI); /// Check whether the dependencies between the accesses are safe. /// diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1243,6 +1243,46 @@ return Diff && *Diff == 1; } +void MemoryDepChecker::addAccess(StoreInst *SI) { + Value *Ptr = SI->getPointerOperand(); + // SCEV does not look through non-header PHIs inside the loop. Such phis + // can be analyzed by adding separate accesses for each incoming pointer + // value. + auto *PN = dyn_cast(Ptr); + if (PN && InnermostLoop->contains(PN->getParent()) && + PN->getParent() != InnermostLoop->getHeader()) { + for (const Use &Inc : PN->incoming_values()) { + Accesses[MemAccessInfo(Inc, true)].push_back(AccessIdx); + InstMap.push_back(SI); + ++AccessIdx; + } + } else { + Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx); + InstMap.push_back(SI); + ++AccessIdx; + } +} + +void MemoryDepChecker::addAccess(LoadInst *LI) { + Value *Ptr = LI->getPointerOperand(); + // SCEV does not look through non-header PHIs inside the loop. Such phis + // can be analyzed by adding separate accesses for each incoming pointer + // value. + auto *PN = dyn_cast(Ptr); + if (PN && InnermostLoop->contains(PN->getParent()) && + PN->getParent() != InnermostLoop->getHeader()) { + for (const Use &Inc : PN->incoming_values()) { + Accesses[MemAccessInfo(Inc, false)].push_back(AccessIdx); + InstMap.push_back(LI); + ++AccessIdx; + } + } else { + Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx); + InstMap.push_back(LI); + ++AccessIdx; + } +} + MemoryDepChecker::VectorizationSafetyStatus MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) { switch (Type) { diff --git a/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll b/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoopDistribute/pointer-phi-in-loop.ll @@ -0,0 +1,153 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -enable-loop-distribute -loop-distribute -S %s | FileCheck %s + +; Testcases inspired by PR50296, PR50288. + +define void @phi_load_store_distribute(i1 %c, i16* %A, i16* %B, i16* %C) { +; CHECK-LABEL: @phi_load_store_distribute( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[IF_END:%.*]] ] +; CHECK-NEXT: [[LV:%.*]] = load i16, i16* [[A:%.*]], align 1 +; CHECK-NEXT: store i16 [[LV]], i16* [[A]], align 1 +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END]] +; CHECK: if.then: +; CHECK-NEXT: [[LV2:%.*]] = load i16, i16* [[A]], align 1 +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[C_SINK:%.*]] = phi i16* [ [[B:%.*]], [[IF_THEN]] ], [ [[C:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: [[LV3:%.*]] = load i16, i16* [[C_SINK]], align 2 +; CHECK-NEXT: [[ADD:%.*]] = add i16 [[LV3]], 10 +; CHECK-NEXT: store i16 [[ADD]], i16* [[C_SINK]], align 1 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i16 [[IV_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]] +; CHECK: for.end.loopexit: +; CHECK-NEXT: ret void +; +entry: + br label %for.body + +for.body: ; preds = %if.end, %entry + %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ] + %lv = load i16, i16* %A, align 1 + store i16 %lv, i16* %A, align 1 + br i1 %c, label %if.then, label %if.end + +if.then: ; preds = %for.body + %lv2 = load i16, i16* %A, align 1 + br label %if.end + +if.end: ; preds = %if.then, %for.body + %c.sink = phi i16* [ %B, %if.then ], [ %C, %for.body ] + %lv3 = load i16, i16* %c.sink + %add = add i16 %lv3, 10 + store i16 %add, i16* %c.sink, align 1 + %iv.next = add nuw nsw i16 %iv, 1 + %tobool.not = icmp eq i16 %iv.next, 1000 + br i1 %tobool.not, label %for.end.loopexit, label %for.body + +for.end.loopexit: ; preds = %if.end + ret void +} + +define void @phi_load_distribute(i1 %c, i16* %A, i16* %B, i16* %C) { +; CHECK-LABEL: @phi_load_distribute( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A1:%.*]] = bitcast i16* [[A:%.*]] to i8* +; CHECK-NEXT: [[B2:%.*]] = bitcast i16* [[B:%.*]] to i8* +; CHECK-NEXT: [[C4:%.*]] = bitcast i16* [[C:%.*]] to i8* +; CHECK-NEXT: br label [[FOR_BODY_LVER_CHECK:%.*]] +; CHECK: for.body.lver.check: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A1]], i64 1 +; CHECK-NEXT: [[UGLYGEP3:%.*]] = getelementptr i8, i8* [[B2]], i64 1 +; CHECK-NEXT: [[UGLYGEP5:%.*]] = getelementptr i8, i8* [[C4]], i64 1 +; CHECK-NEXT: [[BC:%.*]] = bitcast i16* [[A]] to i8* +; CHECK-NEXT: [[BC6:%.*]] = bitcast i16* [[B]] to i8* +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[BC]], [[UGLYGEP3]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[BC6]], [[UGLYGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: [[BC7:%.*]] = bitcast i16* [[A]] to i8* +; CHECK-NEXT: [[BC8:%.*]] = bitcast i16* [[C]] to i8* +; CHECK-NEXT: [[BOUND09:%.*]] = icmp ult i8* [[BC7]], [[UGLYGEP5]] +; CHECK-NEXT: [[BOUND110:%.*]] = icmp ult i8* [[BC8]], [[UGLYGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]] +; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]] +; CHECK-NEXT: [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true +; CHECK-NEXT: br i1 [[MEMCHECK_CONFLICT]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH_LDIST1:%.*]] +; CHECK: for.body.ph.lver.orig: +; CHECK-NEXT: br label [[FOR_BODY_LVER_ORIG:%.*]] +; CHECK: for.body.lver.orig: +; CHECK-NEXT: [[IV_LVER_ORIG:%.*]] = phi i16 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], [[IF_END_LVER_ORIG:%.*]] ] +; CHECK-NEXT: [[LV_LVER_ORIG:%.*]] = load i16, i16* [[A]], align 1 +; CHECK-NEXT: store i16 [[LV_LVER_ORIG]], i16* [[A]], align 1 +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN_LVER_ORIG:%.*]], label [[IF_END_LVER_ORIG]] +; CHECK: if.then.lver.orig: +; CHECK-NEXT: [[LV2_LVER_ORIG:%.*]] = load i16, i16* [[A]], align 1 +; CHECK-NEXT: br label [[IF_END_LVER_ORIG]] +; CHECK: if.end.lver.orig: +; CHECK-NEXT: [[C_SINK_LVER_ORIG:%.*]] = phi i16* [ [[B]], [[IF_THEN_LVER_ORIG]] ], [ [[C]], [[FOR_BODY_LVER_ORIG]] ] +; CHECK-NEXT: [[LV3_LVER_ORIG:%.*]] = load i16, i16* [[C_SINK_LVER_ORIG]], align 2 +; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i16 [[IV_LVER_ORIG]], 1 +; CHECK-NEXT: [[TOBOOL_NOT_LVER_ORIG:%.*]] = icmp eq i16 [[IV_NEXT_LVER_ORIG]], 1000 +; CHECK-NEXT: br i1 [[TOBOOL_NOT_LVER_ORIG]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] +; CHECK: for.body.ph.ldist1: +; CHECK-NEXT: br label [[FOR_BODY_LDIST1:%.*]] +; CHECK: for.body.ldist1: +; CHECK-NEXT: [[IV_LDIST1:%.*]] = phi i16 [ 0, [[FOR_BODY_PH_LDIST1]] ], [ [[IV_NEXT_LDIST1:%.*]], [[IF_END_LDIST1:%.*]] ] +; CHECK-NEXT: [[LV_LDIST1:%.*]] = load i16, i16* [[A]], align 1, !alias.scope !0, !noalias !3 +; CHECK-NEXT: store i16 [[LV_LDIST1]], i16* [[A]], align 1, !alias.scope !0, !noalias !3 +; CHECK-NEXT: br i1 [[C]], label [[IF_THEN_LDIST1:%.*]], label [[IF_END_LDIST1]] +; CHECK: if.then.ldist1: +; CHECK-NEXT: [[LV2_LDIST1:%.*]] = load i16, i16* [[A]], align 1, !alias.scope !0, !noalias !3 +; CHECK-NEXT: br label [[IF_END_LDIST1]] +; CHECK: if.end.ldist1: +; CHECK-NEXT: [[IV_NEXT_LDIST1]] = add nuw nsw i16 [[IV_LDIST1]], 1 +; CHECK-NEXT: [[TOBOOL_NOT_LDIST1:%.*]] = icmp eq i16 [[IV_NEXT_LDIST1]], 1000 +; CHECK-NEXT: br i1 [[TOBOOL_NOT_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]] +; CHECK: for.body.ph: +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 0, [[FOR_BODY_PH]] ], [ [[IV_NEXT:%.*]], [[IF_END:%.*]] ] +; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] +; CHECK: if.then: +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[C_SINK:%.*]] = phi i16* [ [[B]], [[IF_THEN]] ], [ [[C]], [[FOR_BODY]] ] +; CHECK-NEXT: [[LV3:%.*]] = load i16, i16* [[C_SINK]], align 2 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i16 [[IV_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_END_LOOPEXIT_LOOPEXIT12:%.*]], label [[FOR_BODY]] +; CHECK: for.end.loopexit.loopexit: +; CHECK-NEXT: br label [[FOR_END_LOOPEXIT:%.*]] +; CHECK: for.end.loopexit.loopexit12: +; CHECK-NEXT: br label [[FOR_END_LOOPEXIT]] +; CHECK: for.end.loopexit: +; CHECK-NEXT: ret void +; +entry: + br label %for.body + +for.body: ; preds = %if.end, %entry + %iv = phi i16 [ 0, %entry ], [ %iv.next, %if.end ] + %lv = load i16, i16* %A, align 1 + store i16 %lv, i16* %A, align 1 + br i1 %c, label %if.then, label %if.end + +if.then: ; preds = %for.body + %lv2 = load i16, i16* %A, align 1 + br label %if.end + +if.end: ; preds = %if.then, %for.body + %c.sink = phi i16* [ %B, %if.then ], [ %C, %for.body ] + %lv3 = load i16, i16* %c.sink + %iv.next = add nuw nsw i16 %iv, 1 + %tobool.not = icmp eq i16 %iv.next, 1000 + br i1 %tobool.not, label %for.end.loopexit, label %for.body + +for.end.loopexit: ; preds = %if.end + ret void +} + +