Index: llvm/lib/CodeGen/TypePromotion.cpp =================================================================== --- llvm/lib/CodeGen/TypePromotion.cpp +++ llvm/lib/CodeGen/TypePromotion.cpp @@ -18,6 +18,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" @@ -108,7 +109,7 @@ SmallPtrSetImpl &SafeWrap; IntegerType *ExtTy = nullptr; SmallPtrSet NewInsts; - SmallPtrSet InstsToRemove; + SmallPtrSet &InstsToRemove; DenseMap> TruncTysMap; SmallPtrSet Promoted; @@ -123,9 +124,11 @@ IRPromoter(LLVMContext &C, unsigned Width, SetVector &visited, SetVector &sources, SetVector &sinks, - SmallPtrSetImpl &wrap) + SmallPtrSetImpl &wrap, + SmallPtrSet &instsToRemove) : Ctx(C), PromotedWidth(Width), Visited(visited), - Sources(sources), Sinks(sinks), SafeWrap(wrap) { + Sources(sources), Sinks(sinks), SafeWrap(wrap), + InstsToRemove(instsToRemove) { ExtTy = IntegerType::get(Ctx, PromotedWidth); } @@ -139,6 +142,7 @@ SmallPtrSet AllVisited; SmallPtrSet SafeToPromote; SmallPtrSet SafeWrap; + SmallPtrSet InstsToRemove; // Does V have the same size result type as TypeSize. bool EqualTypeSize(Value *V); @@ -174,6 +178,7 @@ TypePromotion() : FunctionPass(ID) {} void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.setPreservesCFG(); @@ -602,7 +607,6 @@ for (auto *I : InstsToRemove) { LLVM_DEBUG(dbgs() << "IR Promotion: Removing " << *I << "\n"); I->dropAllReferences(); - I->eraseFromParent(); } } @@ -870,11 +874,12 @@ // DAG optimizations should be able to handle these cases better, especially // for function arguments. - if (ToPromote < 2 || (Blocks.size() == 1 && (NonFreeArgs > SafeWrap.size()))) + if (!isa(V) + && (ToPromote < 2 || (Blocks.size() == 1 && (NonFreeArgs > SafeWrap.size())))) return false; IRPromoter Promoter(*Ctx, PromotedWidth, CurrentVisited, Sources, Sinks, - SafeWrap); + SafeWrap, InstsToRemove); Promoter.Mutate(); return true; } @@ -899,47 +904,77 @@ const TargetLowering *TLI = SubtargetInfo->getTargetLowering(); const TargetTransformInfo &TII = getAnalysis().getTTI(F); + const LoopInfo &LI = getAnalysis().getLoopInfo(); RegisterBitWidth = TII.getRegisterBitWidth(TargetTransformInfo::RGK_Scalar).getFixedSize(); Ctx = &F.getParent()->getContext(); - // Search up from icmps to try to promote their operands. + // Return the preferred integer width of the instruction, or zero if we + // shouldn't try. + auto GetPromoteWidth = [&](Instruction *I) -> uint32_t { + if (!isa(I->getType())) + return 0; + + EVT SrcVT = TLI->getValueType(DL, I->getType()); + if (SrcVT.isSimple() && TLI->isTypeLegal(SrcVT.getSimpleVT())) + return 0; + + if (TLI->getTypeAction(*Ctx, SrcVT) != TargetLowering::TypePromoteInteger) + return 0; + + EVT PromotedVT = TLI->getTypeToTransformTo(*Ctx, SrcVT); + if (RegisterBitWidth < PromotedVT.getFixedSizeInBits()) { + LLVM_DEBUG(dbgs() << "IR Promotion: Couldn't find target register " + << "for promoted type\n"); + return 0; + } + + // TODO: Should we prefer to use RegisterBitWidth instead? + return PromotedVT.getFixedSizeInBits(); + }; + + auto BBIsInLoop = [&](BasicBlock *BB) -> bool { + for (auto *L : LI) + if (L->contains(BB)) + return true; + return false; + }; + + // Search up from icmps and phis to try to promote their operands. for (BasicBlock &BB : F) { for (Instruction &I : BB) { if (AllVisited.count(&I)) continue; - if (!isa(&I)) - continue; - - auto *ICmp = cast(&I); - // Skip signed or pointer compares - if (ICmp->isSigned() || !isa(ICmp->getOperand(0)->getType())) - continue; - - LLVM_DEBUG(dbgs() << "IR Promotion: Searching from: " << *ICmp << "\n"); - - for (auto &Op : ICmp->operands()) { - if (auto *I = dyn_cast(Op)) { - EVT SrcVT = TLI->getValueType(DL, I->getType()); - if (SrcVT.isSimple() && TLI->isTypeLegal(SrcVT.getSimpleVT())) - break; - - if (TLI->getTypeAction(*Ctx, SrcVT) != - TargetLowering::TypePromoteInteger) - break; - EVT PromotedVT = TLI->getTypeToTransformTo(*Ctx, SrcVT); - if (RegisterBitWidth < PromotedVT.getFixedSizeInBits()) { - LLVM_DEBUG(dbgs() << "IR Promotion: Couldn't find target register " - << "for promoted type\n"); - break; + if (isa(&I) && isa(I.getOperand(0)) && + BBIsInLoop(&BB)) { + LLVM_DEBUG(dbgs() << "IR Promotion: Searching from: " << + I.getOperand(0) << "\n"); + EVT ZExtVT = TLI->getValueType(DL, I.getType()); + Instruction *Phi = dyn_cast(I.getOperand(0)); + MadeChange |= TryToPromote(Phi, ZExtVT.getFixedSizeInBits()); + } else if (auto *ICmp = dyn_cast(&I)) { + // Skip signed or pointer compares + if (ICmp->isSigned()) + continue; + + LLVM_DEBUG(dbgs() << "IR Promotion: Searching from: " << *ICmp << "\n"); + + for (auto &Op : ICmp->operands()) { + if (auto *OpI = dyn_cast(Op)) { + if (auto PromotedWidth = GetPromoteWidth(OpI)) { + MadeChange |= TryToPromote(OpI, PromotedWidth); + break; + } } - - MadeChange |= TryToPromote(I, PromotedVT.getFixedSizeInBits()); - break; } } } + if (!InstsToRemove.empty()) { + for (auto *I : InstsToRemove) + I->eraseFromParent(); + InstsToRemove.clear(); + } } AllVisited.clear(); Index: llvm/test/CodeGen/AArch64/O3-pipeline.ll =================================================================== --- llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -90,8 +90,8 @@ ; CHECK-NEXT: Interleaved Load Combine Pass ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Interleaved Access Pass -; CHECK-NEXT: Type Promotion ; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Type Promotion ; CHECK-NEXT: CodeGen Prepare ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Exception handling preparation Index: llvm/test/Transforms/TypePromotion/AArch64/phi-zext-gep.ll =================================================================== --- llvm/test/Transforms/TypePromotion/AArch64/phi-zext-gep.ll +++ llvm/test/Transforms/TypePromotion/AArch64/phi-zext-gep.ll @@ -24,7 +24,6 @@ ; CHECK-NEXT: br label [[FOR_END]] ; CHECK: for.end: ; CHECK-NEXT: [[TAG_0_IN_LCSSA:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP1]], [[FOR_END_LOOPEXIT]] ] -; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TAG_0_IN_LCSSA]] to i8 ; CHECK-NEXT: ret i32 [[TAG_0_IN_LCSSA]] ; entry: Index: llvm/test/Transforms/TypePromotion/AArch64/phi-zext-gep2.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/TypePromotion/AArch64/phi-zext-gep2.ll @@ -0,0 +1,44 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=aarch64 -type-promotion -verify -S %s -o - | FileCheck %s +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + +; Function Attrs: mustprogress nofree nosync nounwind uwtable +define dso_local void @foo(ptr noundef %ptr0, ptr nocapture noundef readonly %ptr1, ptr nocapture noundef %dest) local_unnamed_addr { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[PTR0:%.*]], align 1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i64 +; CHECK-NEXT: br label [[DO_BODY:%.*]] +; CHECK: do.body: +; CHECK-NEXT: [[TO_PROMOTE:%.*]] = phi i64 [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP4:%.*]], [[DO_BODY]] ] +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[PTR1:%.*]], i64 [[TO_PROMOTE]] +; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 2 +; CHECK-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i64 +; CHECK-NEXT: [[COND_IN_I:%.*]] = getelementptr inbounds i8, ptr [[PTR1]], i64 [[TMP3]] +; CHECK-NEXT: [[COND_I:%.*]] = load i8, ptr [[COND_IN_I]], align 1 +; CHECK-NEXT: [[TMP4]] = zext i8 [[COND_I]] to i64 +; CHECK-NEXT: store i8 [[TMP2]], ptr [[DEST:%.*]], align 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[TMP4]], 0 +; CHECK-NEXT: br i1 [[CMP]], label [[DO_BODY]], label [[DO_END:%.*]] +; CHECK: do.end: +; CHECK-NEXT: ret void +; +entry: + %0 = load i8, ptr %ptr0, align 1 + br label %do.body + +do.body: ; preds = %do.body, %entry + %to_promote = phi i8 [ %0, %entry ], [ %cond.i, %do.body ] + %ext0 = zext i8 %to_promote to i64 + %arrayidx1 = getelementptr inbounds i8, ptr %ptr1, i64 %ext0 + %1 = load i8, ptr %arrayidx1, align 2 + %2 = zext i8 %1 to i64 + %cond.in.i = getelementptr inbounds i8, ptr %ptr1, i64 %2 + %cond.i = load i8, ptr %cond.in.i, align 1 + store i8 %1, ptr %dest, align 1 + %cmp = icmp ult i8 %cond.i, 0 + br i1 %cmp, label %do.body, label %do.end + +do.end: ; preds = %do.body + ret void +}