diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -415,6 +415,10 @@ /// Return true if the only users of this pointer are lifetime markers. bool onlyUsedByLifetimeMarkers(const Value *V); + /// Return true if the only users of this pointer are lifetime markers or + /// droppable instructions. + bool onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V); + /// Return true if speculation of the given load must be suppressed to avoid /// ordering or interfering with an active sanitizer. If not suppressed, /// dereferenceability and alignment must be proven separately. Note: This diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -4295,18 +4295,33 @@ return true; } -/// Return true if the only users of this pointer are lifetime markers. -bool llvm::onlyUsedByLifetimeMarkers(const Value *V) { +static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper( + const Value *V, bool AllowLifetime, bool AllowDroppable) { for (const User *U : V->users()) { const IntrinsicInst *II = dyn_cast(U); - if (!II) return false; - - if (!II->isLifetimeStartOrEnd()) + if (!II) return false; + + if (AllowLifetime && II->isLifetimeStartOrEnd()) + continue; + + if (AllowDroppable && II->isDroppable()) + continue; + + return false; } return true; } +bool llvm::onlyUsedByLifetimeMarkers(const Value *V) { + return onlyUsedByLifetimeMarkersOrDroppableInstsHelper( + V, /* AllowLifetime */ true, /* AllowDroppable */ false); +} +bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) { + return onlyUsedByLifetimeMarkersOrDroppableInstsHelper( + V, /* AllowLifetime */ true, /* AllowDroppable */ true); +} + bool llvm::mustSuppressSpeculation(const LoadInst &LI) { if (!LI.isUnordered()) return true; diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp --- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -62,10 +62,6 @@ STATISTIC(NumPHIInsert, "Number of PHI nodes inserted"); bool llvm::isAllocaPromotable(const AllocaInst *AI) { - // FIXME: If the memory unit is of pointer or integer type, we can permit - // assignments to subsections of the memory unit. - unsigned AS = AI->getType()->getAddressSpace(); - // Only allow direct and non-volatile loads and stores... for (const User *U : AI->users()) { if (const LoadInst *LI = dyn_cast(U)) { @@ -81,19 +77,15 @@ if (SI->isVolatile()) return false; } else if (const IntrinsicInst *II = dyn_cast(U)) { - if (!II->isLifetimeStartOrEnd()) + if (!II->isLifetimeStartOrEnd() && !II->isDroppable()) return false; } else if (const BitCastInst *BCI = dyn_cast(U)) { - if (BCI->getType() != Type::getInt8PtrTy(U->getContext(), AS)) - return false; - if (!onlyUsedByLifetimeMarkers(BCI)) + if (!onlyUsedByLifetimeMarkersOrDroppableInsts(BCI)) return false; } else if (const GetElementPtrInst *GEPI = dyn_cast(U)) { - if (GEPI->getType() != Type::getInt8PtrTy(U->getContext(), AS)) - return false; if (!GEPI->hasAllZeroIndices()) return false; - if (!onlyUsedByLifetimeMarkers(GEPI)) + if (!onlyUsedByLifetimeMarkersOrDroppableInsts(GEPI)) return false; } else { return false; @@ -312,7 +304,7 @@ AC->registerAssumption(CI); } -static void removeLifetimeIntrinsicUsers(AllocaInst *AI) { +static void removeIntrinsicUsers(AllocaInst *AI) { // Knowing that this alloca is promotable, we know that it's safe to kill all // instructions except for load and store. @@ -322,6 +314,21 @@ if (isa(I) || isa(I)) continue; + // Drop the use of AI in droppable instructions. + if (I->isDroppable()) { + // TODO For now we forget assumed information, this can be improved. + assert(isa(I) && + cast(I)->getIntrinsicID() == Intrinsic::assume && + "Expected assume"); + + // Skip ahead if I has multiple uses of AI. + while (UI != UE && *UI == I) + ++UI; + + I->replaceUsesOfWith(AI, UndefValue::get(AI->getType())); + continue; + } + if (!I->getType()->isVoidTy()) { // The only users of this bitcast/GEP instruction are lifetime intrinsics. // Follow the use/def chain to erase them now instead of leaving it for @@ -329,6 +336,16 @@ for (auto UUI = I->user_begin(), UUE = I->user_end(); UUI != UUE;) { Instruction *Inst = cast(*UUI); ++UUI; + + // Drop the use of I in droppable instructions. + if (Inst->isDroppable()) { + // Skip ahead if I has multiple uses of AI. + while (UUI != UUE && *UUI == Inst) + ++UUI; + + Inst->replaceUsesOfWith(I, UndefValue::get(I->getType())); + continue; + } Inst->eraseFromParent(); } } @@ -544,7 +561,7 @@ assert(AI->getParent()->getParent() == &F && "All allocas should be in the same function, which is same as DF!"); - removeLifetimeIntrinsicUsers(AI); + removeIntrinsicUsers(AI); if (AI->use_empty()) { // If there are no uses of the alloca, just delete it now. diff --git a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll @@ -0,0 +1,84 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mem2reg -S -o - < %s | FileCheck %s + +declare void @llvm.assume(i1) +declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr) +declare void @llvm.lifetime.end.p0i8(i64 %size, i8* nocapture %ptr) + +define void @positive_assume_uses(i32* %arg) { +; CHECK-LABEL: @positive_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[ARG:%.*]]), "align"(i32* undef, i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i32* undef, i64 8), "nonnull"(i32* [[ARG]]) ] +; CHECK-NEXT: ret void +; + %A = alloca i32 + call void @llvm.assume(i1 true) ["nonnull"(i32* %arg), "align"(i32* %A, i64 2)] + store i32 1, i32* %A + call void @llvm.assume(i1 true) ["align"(i32* %A, i64 8), "nonnull"(i32* %arg)] + ret void +} + +define void @negative_assume_condition_use() { +; CHECK-LABEL: @negative_assume_condition_use( +; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[B:%.*]] = bitcast i32* [[A]] to i8* +; CHECK-NEXT: [[CND:%.*]] = icmp eq i8* [[B]], null +; CHECK-NEXT: call void @llvm.assume(i1 [[CND]]) +; CHECK-NEXT: store i32 1, i32* [[A]], align 4 +; CHECK-NEXT: ret void +; + %A = alloca i32 + %B = bitcast i32* %A to i8* + %cnd = icmp eq i8* %B, null + call void @llvm.assume(i1 %cnd) + store i32 1, i32* %A + ret void +} + +define void @positive_multiple_assume_uses() { +; CHECK-LABEL: @positive_multiple_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"({ i8, i16 }* undef, i64 8), "align"({ i8, i16 }* undef, i64 16) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"({ i8, i16 }* undef), "align"({ i8, i16 }* undef, i64 2) ] +; CHECK-NEXT: ret void +; + %A = alloca {i8, i16} + call void @llvm.assume(i1 true) ["align"({i8, i16}* %A, i64 8), "align"({i8, i16}* %A, i64 16)] + store {i8, i16} zeroinitializer, {i8, i16}* %A + call void @llvm.assume(i1 true) ["nonnull"({i8, i16}* %A), "align"({i8, i16}* %A, i64 2)] + ret void +} + +define void @positive_gep_assume_uses() { +; CHECK-LABEL: @positive_gep_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* undef, i64 8), "align"(i8* undef, i64 16) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i8* undef), "align"(i8* undef, i64 2) ] +; CHECK-NEXT: ret void +; + %A = alloca {i8, i16} + %B = getelementptr {i8, i16}, {i8, i16}* %A, i32 0, i32 0 + call void @llvm.lifetime.start.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["align"(i8* %B, i64 8), "align"(i8* %B, i64 16)] + store {i8, i16} zeroinitializer, {i8, i16}* %A + call void @llvm.lifetime.end.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %B, i64 2)] + ret void +} + +define void @positive_mixed_assume_uses() { +; CHECK-LABEL: @positive_mixed_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i8* undef), "align"(i8* undef, i64 8), "align"(i8* undef, i64 16) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i8* undef), "align"(i8* undef, i64 2), "nonnull"(i8* undef) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* undef), "align"(i32* undef, i64 2), "nonnull"(i8* undef) ] +; CHECK-NEXT: ret void +; + %A = alloca i8 + %B = getelementptr i8, i8* %A, i32 0 + %C = bitcast i8* %A to i32* + call void @llvm.lifetime.start.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 8), "align"(i8* %B, i64 16)] + store i8 1, i8* %A + call void @llvm.lifetime.end.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 2), "nonnull"(i8* %A)] + call void @llvm.assume(i1 true) ["nonnull"(i32* %C), "align"(i32* %C, i64 2), "nonnull"(i8* %A)] + ret void +}