[AMDGPUCodegenPrepare] Add NewPM Support

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

1884

I'd assume we could preserve some of these

[gandhi21299]

• Right, I believe all analyses are preserved @arsenm

[aeubanks]

In D151241#4366094, @gandhi21299 wrote:

• Right, I believe all analyses are preserved @arsenm

definitely doesn't look like all analyses are preserved, the pass adds instructions
if the pass doesn't modify the control flow graph, then you can preserve CFGAnalyses

[gandhi21299]

• Preserve CFGAnalyses if the function was modified (asserting no structural changes). Preserve all analyses otherwise.

[gandhi21299]

• rebase against main

[aeubanks]

lgtm (I didn't verify that the pass actually doesn't make CFG changes)

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

1866–1867

put this in the if block

1868–1870

extraneous empty line

[gandhi21299]

• Suggested edits

All insertions/deletions are of instructions and none of them are related to branch instructions. Hence, CFGAnalyses are preserved.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

1868

The CFG does change if the div expansion happens

[gandhi21299]

Preserve TargetLibraryAnalysis and AssumptionAnalysis in all cases. If no changes are made by the pass, preserve all analyses. Otherwise preserve the two only.

[aeubanks]

In D151241#4369287, @gandhi21299 wrote:

Preserve TargetLibraryAnalysis and AssumptionAnalysis in all cases. If no changes are made by the pass, preserve all analyses. Otherwise preserve the two only.

there is no need to preserve those two, they are always preserved (see the invalidate method) even if you don't preserve all analyses

[gandhi21299]

• Revert back to rev-1

[gandhi21299]

^

[aeubanks]

you could keep track of if you've made CFG changes, but this still lgtm

[gandhi21299]

I only found one place in the code where there may be structural changes in the CFG.

[aeubanks]

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

1885

!FlowChanged?

[gandhi21299]

• Added !

llvm/lib/Target/AMDGPU/AMDGPU.h

 struct AMDGPUAlwaysInlinePass : PassInfoMixin<AMDGPUAlwaysInlinePass> {
   AMDGPUAlwaysInlinePass(bool GlobalOpt = true) : GlobalOpt(GlobalOpt) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
 private:
   bool GlobalOpt;
 };
 
+class AMDGPUCodeGenPreparePass
+    : public PassInfoMixin<AMDGPUCodeGenPreparePass> {
+private:
+  TargetMachine &TM;
+
+public:
+  AMDGPUCodeGenPreparePass(TargetMachine &TM) : TM(TM){};
+  PreservedAnalyses run(Function &, FunctionAnalysisManager &);
+};
+
 FunctionPass *createAMDGPUAnnotateUniformValues();
 
 ModulePass *createAMDGPUPrintfRuntimeBinding();
 void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry&);
 extern char &AMDGPUPrintfRuntimeBindingID;
 
 void initializeAMDGPUResourceUsageAnalysisPass(PassRegistry &);
 extern char &AMDGPUResourceUsageAnalysisID;

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

 // Leave all division operations as they are. This supersedes ExpandDiv64InIR
 // and is used for testing the legalizer.
 static cl::opt<bool> DisableIDivExpand(
   "amdgpu-codegenprepare-disable-idiv-expansion",
   cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"),
   cl::ReallyHidden,
   cl::init(false));
 
-class AMDGPUCodeGenPrepare : public FunctionPass,
-                             public InstVisitor<AMDGPUCodeGenPrepare, bool> {
+class AMDGPUCodeGenPrepareImpl
+    : public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
+public:
   const GCNSubtarget *ST = nullptr;
   const TargetLibraryInfo *TLInfo = nullptr;
   AssumptionCache *AC = nullptr;
   DominatorTree *DT = nullptr;
   UniformityInfo *UA = nullptr;
   Module *Mod = nullptr;
   const DataLayout *DL = nullptr;
   bool HasUnsafeFPMath = false;
   bool HasFP32Denormals = false;
+  bool FlowChanged = false;
 
   DenseMap<const PHINode *, bool> BreakPhiNodesCache;
 
   bool canBreakPHINode(const PHINode &I);
 
   /// Copies exact/nsw/nuw flags (if any) from binary operation \p I to
   /// binary operation \p V.
   ///
public:
   /// \returns True.
 
   bool canWidenScalarExtLoad(LoadInst &I) const;
 
   Value *matchFractPat(IntrinsicInst &I);
   Value *applyFractPat(IRBuilder<> &Builder, Value *FractArg);
 
 public:
-  static char ID;
-
-  AMDGPUCodeGenPrepare() : FunctionPass(ID) {}
-
   bool visitFDiv(BinaryOperator &I);
   bool visitXor(BinaryOperator &I);
 
   bool visitInstruction(Instruction &I) { return false; }
   bool visitBinaryOperator(BinaryOperator &I);
   bool visitLoadInst(LoadInst &I);
   bool visitICmpInst(ICmpInst &I);
   bool visitSelectInst(SelectInst &I);
   bool visitPHINode(PHINode &I);
 
   bool visitIntrinsicInst(IntrinsicInst &I);
   bool visitBitreverseIntrinsicInst(IntrinsicInst &I);
   bool visitMinNum(IntrinsicInst &I);
+  bool run(Function &F);
+};
 
-  bool doInitialization(Module &M) override;
-  bool runOnFunction(Function &F) override;
-
-  StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
+class AMDGPUCodeGenPrepare : public FunctionPass {
+private:
+  AMDGPUCodeGenPrepareImpl Impl;
 
+public:
+  static char ID;
+  AMDGPUCodeGenPrepare() : FunctionPass(ID) {
+    initializeAMDGPUCodeGenPreparePass(*PassRegistry::getPassRegistry());
+  }
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<UniformityInfoWrapperPass>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
 
     // FIXME: Division expansion needs to preserve the dominator tree.
     if (!ExpandDiv64InIR)
       AU.setPreservesAll();
   }
+  bool runOnFunction(Function &F) override;
+  bool doInitialization(Module &M) override;
+  StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
 };
 
 } // end anonymous namespace
 
-unsigned AMDGPUCodeGenPrepare::getBaseElementBitWidth(const Type *T) const {
+bool AMDGPUCodeGenPrepareImpl::run(Function &F) {
+  bool MadeChange = false;
+
+  Function::iterator NextBB;
+  for (Function::iterator FI = F.begin(), FE = F.end(); FI != FE; FI = NextBB) {
+    BasicBlock *BB = &*FI;
+    NextBB = std::next(FI);
+
+    BasicBlock::iterator Next;
+    for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;
+         I = Next) {
+      Next = std::next(I);
+
+      MadeChange |= visit(*I);
+
+      if (Next != E) { // Control flow changed
+        BasicBlock *NextInstBB = Next->getParent();
+        if (NextInstBB != BB) {
+          BB = NextInstBB;
+          E = BB->end();
+          FE = F.end();
+        }
+      }
+    }
+  }
+  return MadeChange;
+}
+
+unsigned AMDGPUCodeGenPrepareImpl::getBaseElementBitWidth(const Type *T) const {
   assert(needsPromotionToI32(T) && "T does not need promotion to i32");
 
   if (T->isIntegerTy())
     return T->getIntegerBitWidth();
   return cast<VectorType>(T)->getElementType()->getIntegerBitWidth();
 }
 
-Type *AMDGPUCodeGenPrepare::getI32Ty(IRBuilder<> &B, const Type *T) const {
+Type *AMDGPUCodeGenPrepareImpl::getI32Ty(IRBuilder<> &B, const Type *T) const {
   assert(needsPromotionToI32(T) && "T does not need promotion to i32");
 
   if (T->isIntegerTy())
     return B.getInt32Ty();
   return FixedVectorType::get(B.getInt32Ty(), cast<FixedVectorType>(T));
 }
 
-bool AMDGPUCodeGenPrepare::isSigned(const BinaryOperator &I) const {
+bool AMDGPUCodeGenPrepareImpl::isSigned(const BinaryOperator &I) const {
   return I.getOpcode() == Instruction::AShr ||
       I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::SRem;
 }
 
-bool AMDGPUCodeGenPrepare::isSigned(const SelectInst &I) const {
+bool AMDGPUCodeGenPrepareImpl::isSigned(const SelectInst &I) const {
   return isa<ICmpInst>(I.getOperand(0)) ?
       cast<ICmpInst>(I.getOperand(0))->isSigned() : false;
 }
 
-bool AMDGPUCodeGenPrepare::needsPromotionToI32(const Type *T) const {
+bool AMDGPUCodeGenPrepareImpl::needsPromotionToI32(const Type *T) const {
   if (!Widen16BitOps)
     return false;
 
   const IntegerType *IntTy = dyn_cast<IntegerType>(T);
   if (IntTy && IntTy->getBitWidth() > 1 && IntTy->getBitWidth() <= 16)
     return true;
 
   if (const VectorType *VT = dyn_cast<VectorType>(T)) {
     // TODO: The set of packed operations is more limited, so may want to
     // promote some anyway.
     if (ST->hasVOP3PInsts())
       return false;
 
     return needsPromotionToI32(VT->getElementType());
   }
 
   return false;
 }
 
-bool AMDGPUCodeGenPrepare::isLegalFloatingTy(const Type *Ty) const {
+bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const {
   return Ty->isFloatTy() || Ty->isDoubleTy() ||
          (Ty->isHalfTy() && ST->has16BitInsts());
 }
 
 // Return true if the op promoted to i32 should have nsw set.
 static bool promotedOpIsNSW(const Instruction &I) {
   switch (I.getOpcode()) {
   case Instruction::Shl:
case Instruction::Mul:
     return true;
   case Instruction::Sub:
     return I.hasNoUnsignedWrap();
   default:
     return false;
   }
 }
 
-bool AMDGPUCodeGenPrepare::canWidenScalarExtLoad(LoadInst &I) const {
+bool AMDGPUCodeGenPrepareImpl::canWidenScalarExtLoad(LoadInst &I) const {
   Type *Ty = I.getType();
   const DataLayout &DL = Mod->getDataLayout();
   int TySize = DL.getTypeSizeInBits(Ty);
   Align Alignment = DL.getValueOrABITypeAlignment(I.getAlign(), Ty);
 
   return I.isSimple() && TySize < 32 && Alignment >= 4 && UA->isUniform(&I);
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(BinaryOperator &I) const {
+bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(BinaryOperator &I) const {
   assert(needsPromotionToI32(I.getType()) &&
          "I does not need promotion to i32");
 
   if (I.getOpcode() == Instruction::SDiv ||
       I.getOpcode() == Instruction::UDiv ||
       I.getOpcode() == Instruction::SRem ||
       I.getOpcode() == Instruction::URem)
     return false;
bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(BinaryOperator &I) const {
   TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
 
   I.replaceAllUsesWith(TruncRes);
   I.eraseFromParent();
 
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(ICmpInst &I) const {
+bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(ICmpInst &I) const {
   assert(needsPromotionToI32(I.getOperand(0)->getType()) &&
          "I does not need promotion to i32");
 
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
   Type *I32Ty = getI32Ty(Builder, I.getOperand(0)->getType());
   Value *ExtOp0 = nullptr;
bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(ICmpInst &I) const {
   NewICmp = Builder.CreateICmp(I.getPredicate(), ExtOp0, ExtOp1);
 
   I.replaceAllUsesWith(NewICmp);
   I.eraseFromParent();
 
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(SelectInst &I) const {
+bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(SelectInst &I) const {
   assert(needsPromotionToI32(I.getType()) &&
          "I does not need promotion to i32");
 
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
   Type *I32Ty = getI32Ty(Builder, I.getType());
   Value *ExtOp1 = nullptr;
bool AMDGPUCodeGenPrepareImpl::promoteUniformOpToI32(SelectInst &I) const {
   TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
 
   I.replaceAllUsesWith(TruncRes);
   I.eraseFromParent();
 
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::promoteUniformBitreverseToI32(
+bool AMDGPUCodeGenPrepareImpl::promoteUniformBitreverseToI32(
     IntrinsicInst &I) const {
   assert(I.getIntrinsicID() == Intrinsic::bitreverse &&
          "I must be bitreverse intrinsic");
   assert(needsPromotionToI32(I.getType()) &&
          "I does not need promotion to i32");
 
   IRBuilder<> Builder(&I);
   Builder.SetCurrentDebugLocation(I.getDebugLoc());
Value *TruncRes =
       Builder.CreateTrunc(LShrOp, I.getType());
 
   I.replaceAllUsesWith(TruncRes);
   I.eraseFromParent();
 
   return true;
 }
 
-unsigned AMDGPUCodeGenPrepare::numBitsUnsigned(Value *Op) const {
+unsigned AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op) const {
   return computeKnownBits(Op, *DL, 0, AC).countMaxActiveBits();
 }
 
-unsigned AMDGPUCodeGenPrepare::numBitsSigned(Value *Op) const {
+unsigned AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op) const {
   return ComputeMaxSignificantBits(Op, *DL, 0, AC);
 }
 
 static void extractValues(IRBuilder<> &Builder,
                           SmallVectorImpl<Value *> &Values, Value *V) {
   auto *VT = dyn_cast<FixedVectorType>(V->getType());
   if (!VT) {
     Values.push_back(V);
static Value *getMul24(IRBuilder<> &Builder, Value *LHS, Value *RHS,
 
   IntegerType *I64Ty = Builder.getInt64Ty();
   Lo = Builder.CreateZExtOrTrunc(Lo, I64Ty);
   Hi = Builder.CreateZExtOrTrunc(Hi, I64Ty);
 
   return Builder.CreateOr(Lo, Builder.CreateShl(Hi, 32));
 }
 
-bool AMDGPUCodeGenPrepare::replaceMulWithMul24(BinaryOperator &I) const {
+bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
   if (I.getOpcode() != Instruction::Mul)
     return false;
 
   Type *Ty = I.getType();
   unsigned Size = Ty->getScalarSizeInBits();
   if (Size <= 16 && ST->has16BitInsts())
     return false;
 
static SelectInst *findSelectThroughCast(Value *V, CastInst *&Cast) {
   if ((Cast = dyn_cast<CastInst>(V))) {
     if (SelectInst *Sel = dyn_cast<SelectInst>(Cast->getOperand(0)))
       return Sel;
   }
 
   return nullptr;
 }
 
-bool AMDGPUCodeGenPrepare::foldBinOpIntoSelect(BinaryOperator &BO) const {
+bool AMDGPUCodeGenPrepareImpl::foldBinOpIntoSelect(BinaryOperator &BO) const {
   // Don't do this unless the old select is going away. We want to eliminate the
   // binary operator, not replace a binop with a select.
   int SelOpNo = 0;
 
   CastInst *CastOp;
 
   // TODO: Should probably try to handle some cases with multiple
   // users. Duplicating the select may be profitable for division.

 //   a/b -> a*rcp(b) when inaccurate rcp is allowed with unsafe-fp-math or afn.
 //
 // With fdiv.fast:
 //   a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
 //
 //   1/x -> fdiv.fast(1,x)  when !fpmath >= 2.5ulp.
 //
 // NOTE: rcp is the preference in cases that both are legal.
-bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) {
+bool AMDGPUCodeGenPrepareImpl::visitFDiv(BinaryOperator &FDiv) {
 
   Type *Ty = FDiv.getType()->getScalarType();
 
   // The f64 rcp/rsq approximations are pretty inaccurate. We can do an
   // expansion around them in codegen.
   if (Ty->isDoubleTy())
     return false;
 
if (NewFDiv) {
     FDiv.replaceAllUsesWith(NewFDiv);
     NewFDiv->takeName(&FDiv);
     FDiv.eraseFromParent();
   }
 
   return !!NewFDiv;
 }
 
-bool AMDGPUCodeGenPrepare::visitXor(BinaryOperator &I) {
+bool AMDGPUCodeGenPrepareImpl::visitXor(BinaryOperator &I) {
   // Match the Xor instruction, its type and its operands
   IntrinsicInst *IntrinsicCall = dyn_cast<IntrinsicInst>(I.getOperand(0));
   ConstantInt *RHS = dyn_cast<ConstantInt>(I.getOperand(1));
   if (!RHS || !IntrinsicCall || RHS->getSExtValue() != -1)
     return visitBinaryOperator(I);
 
   // Check if the Call is an intrinsic instruction to amdgcn_class intrinsic
   // has only one use

 
 static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
   return getMul64(Builder, LHS, RHS).second;
 }
 
 /// Figure out how many bits are really needed for this division. \p AtLeast is
 /// an optimization hint to bypass the second ComputeNumSignBits call if we the
 /// first one is insufficient. Returns -1 on failure.
-int AMDGPUCodeGenPrepare::getDivNumBits(BinaryOperator &I,
-                                        Value *Num, Value *Den,
-                                        unsigned AtLeast, bool IsSigned) const {
+int AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
+                                            Value *Den, unsigned AtLeast,
+                                            bool IsSigned) const {
   const DataLayout &DL = Mod->getDataLayout();
   unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
   if (LHSSignBits < AtLeast)
     return -1;
 
   unsigned RHSSignBits = ComputeNumSignBits(Den, DL, 0, AC, &I);
   if (RHSSignBits < AtLeast)
     return -1;
 
   unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
   unsigned DivBits = Num->getType()->getScalarSizeInBits() - SignBits;
   if (IsSigned)
     ++DivBits;
   return DivBits;
 }
 
 // The fractional part of a float is enough to accurately represent up to
 // a 24-bit signed integer.
-Value *AMDGPUCodeGenPrepare::expandDivRem24(IRBuilder<> &Builder,
-                                            BinaryOperator &I,
-                                            Value *Num, Value *Den,
-                                            bool IsDiv, bool IsSigned) const {
+Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder,
+                                                BinaryOperator &I, Value *Num,
+                                                Value *Den, bool IsDiv,
+                                                bool IsSigned) const {
   int DivBits = getDivNumBits(I, Num, Den, 9, IsSigned);
   if (DivBits == -1)
     return nullptr;
   return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned);
 }
 
-Value *AMDGPUCodeGenPrepare::expandDivRem24Impl(IRBuilder<> &Builder,
-                                                BinaryOperator &I,
-                                                Value *Num, Value *Den,
-                                                unsigned DivBits,
-                                                bool IsDiv, bool IsSigned) const {
+Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl(
+    IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den,
+    unsigned DivBits, bool IsDiv, bool IsSigned) const {
   Type *I32Ty = Builder.getInt32Ty();
   Num = Builder.CreateTrunc(Num, I32Ty);
   Den = Builder.CreateTrunc(Den, I32Ty);
 
   Type *F32Ty = Builder.getFloatTy();
   ConstantInt *One = Builder.getInt32(1);
   Value *JQ = One;
 
Value *AMDGPUCodeGenPrepareImpl::expandDivRem24Impl(
 
   return Res;
 }
 
 // Try to recognize special cases the DAG will emit special, better expansions
 // than the general expansion we do here.
 
 // TODO: It would be better to just directly handle those optimizations here.
-bool AMDGPUCodeGenPrepare::divHasSpecialOptimization(
-  BinaryOperator &I, Value *Num, Value *Den) const {
+bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
+                                                         Value *Num,
+                                                         Value *Den) const {
   if (Constant *C = dyn_cast<Constant>(Den)) {
     // Arbitrary constants get a better expansion as long as a wider mulhi is
     // legal.
     if (C->getType()->getScalarSizeInBits() <= 32)
       return true;
 
     // TODO: Sdiv check for not exact for some reason.
 
static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout *DL) {
   KnownBits Known = computeKnownBits(V, *DL);
   if (Known.isNegative())
     return Constant::getAllOnesValue(V->getType());
   if (Known.isNonNegative())
     return Constant::getNullValue(V->getType());
   return Builder.CreateAShr(V, Builder.getInt32(31));
 }
 
-Value *AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
+Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
                                                 BinaryOperator &I, Value *X,
                                                 Value *Y) const {
   Instruction::BinaryOps Opc = I.getOpcode();
   assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
          Opc == Instruction::SRem || Opc == Instruction::SDiv);
 
   FastMathFlags FMF;
   FMF.setFast();
   Builder.setFastMathFlags(FMF);
 
if (IsSigned) {
     Res = Builder.CreateSub(Res, Sign);
   }
 
   Res = Builder.CreateTrunc(Res, Ty);
 
   return Res;
 }
 
-Value *AMDGPUCodeGenPrepare::shrinkDivRem64(IRBuilder<> &Builder,
-                                            BinaryOperator &I,
-                                            Value *Num, Value *Den) const {
+Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
+                                                BinaryOperator &I, Value *Num,
+                                                Value *Den) const {
   if (!ExpandDiv64InIR && divHasSpecialOptimization(I, Num, Den))
     return nullptr;  // Keep it for later optimization.
 
   Instruction::BinaryOps Opc = I.getOpcode();
 
   bool IsDiv = Opc == Instruction::SDiv || Opc == Instruction::UDiv;
   bool IsSigned = Opc == Instruction::SDiv || Opc == Instruction::SRem;
 
Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
   if (Narrowed) {
     return IsSigned ? Builder.CreateSExt(Narrowed, Num->getType()) :
                       Builder.CreateZExt(Narrowed, Num->getType());
   }
 
   return nullptr;
 }
 
-void AMDGPUCodeGenPrepare::expandDivRem64(BinaryOperator &I) const {
+void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
   Instruction::BinaryOps Opc = I.getOpcode();
   // Do the general expansion.
   if (Opc == Instruction::UDiv || Opc == Instruction::SDiv) {
     expandDivisionUpTo64Bits(&I);
     return;
   }
 
   if (Opc == Instruction::URem || Opc == Instruction::SRem) {
     expandRemainderUpTo64Bits(&I);
     return;
   }
 
   llvm_unreachable("not a division");
 }
 
-bool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) {
+bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
   if (foldBinOpIntoSelect(I))
     return true;
 
   if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
       UA->isUniform(&I) && promoteUniformOpToI32(I))
     return true;
 
   if (UseMul24Intrin && replaceMulWithMul24(I))
if (NewDiv) {
       Changed = true;
     }
   }
 
   if (ExpandDiv64InIR) {
     // TODO: We get much worse code in specially handled constant cases.
     for (BinaryOperator *Div : Div64ToExpand) {
       expandDivRem64(*Div);
+      FlowChanged = true;
       Changed = true;
     }
   }
 
   return Changed;
 }
 
-bool AMDGPUCodeGenPrepare::visitLoadInst(LoadInst &I) {
+bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
   if (!WidenLoads)
     return false;
 
   if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
        I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
       canWidenScalarExtLoad(I)) {
     IRBuilder<> Builder(&I);
     Builder.SetCurrentDebugLocation(I.getDebugLoc());
if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
     I.replaceAllUsesWith(ValOrig);
     I.eraseFromParent();
     return true;
   }
 
   return false;
 }
 
-bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) {
+bool AMDGPUCodeGenPrepareImpl::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
 
   if (ST->has16BitInsts() && needsPromotionToI32(I.getOperand(0)->getType()) &&
       UA->isUniform(&I))
     Changed |= promoteUniformOpToI32(I);
 
   return Changed;
 }
 
-bool AMDGPUCodeGenPrepare::visitSelectInst(SelectInst &I) {
-  if (ST->has16BitInsts() && needsPromotionToI32(I.getType())) {
-    if (UA->isUniform(&I))
-      return promoteUniformOpToI32(I);
-    return false;
-  }
-
+bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
   Value *Cond = I.getCondition();
   Value *TrueVal = I.getTrueValue();
   Value *FalseVal = I.getFalseValue();
   Value *CmpVal;
   FCmpInst::Predicate Pred;
 
+  if (ST->has16BitInsts() && needsPromotionToI32(I.getType())) {
+    if (UA->isUniform(&I))
+      return promoteUniformOpToI32(I);
+    return false;
+  }
+
   // Match fract pattern with nan check.
   if (!match(Cond, m_FCmp(Pred, m_Value(CmpVal), m_NonNaN())))
     return false;
 
   FPMathOperator *FPOp = dyn_cast<FPMathOperator>(&I);
   if (!FPOp)
     return false;
 
if (const auto *SV = dyn_cast<ShuffleVectorInst>(CurVal)) {
     return isa<Constant>(SV->getOperand(1)) ||
            areInSameBB(SV, SV->getOperand(0)) ||
            areInSameBB(SV, SV->getOperand(1));
   }
 
   return false;
 }
 
-bool AMDGPUCodeGenPrepare::canBreakPHINode(const PHINode &I) {
+bool AMDGPUCodeGenPrepareImpl::canBreakPHINode(const PHINode &I) {
   // Check in the cache, or add an entry for this node.
   //
   // We init with false because we consider all PHI nodes unbreakable until we
   // reach a conclusion. Doing the opposite - assuming they're break-able until
   // proven otherwise - can be harmful in some pathological cases so we're
   // conservative for now.
   const auto [It, DidInsert] = BreakPhiNodesCache.insert({&I, false});
   if (!DidInsert)
Value *getSlicedVal(BasicBlock *BB, Value *Inc, StringRef NewValName) {
 
     return Res;
   }
 
 private:
   SmallDenseMap<std::pair<BasicBlock *, Value *>, Value *> SlicedVals;
 };
 
-bool AMDGPUCodeGenPrepare::visitPHINode(PHINode &I) {
+bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
   // Break-up fixed-vector PHIs into smaller pieces.
   // Default threshold is 32, so it breaks up any vector that's >32 bits into
   // its elements, or into 32-bit pieces (for 8/16 bit elts).
   //
   // This is only helpful for DAGISel because it doesn't handle large PHIs as
   // well as GlobalISel. DAGISel lowers PHIs by using CopyToReg/CopyFromReg.
   // With large, odd-sized PHIs we may end up needing many `build_vector`
   // operations with most elements being "undef". This inhibits a lot of
else
       Vec = B.CreateInsertElement(Vec, S.NewPHI, S.Idx, ValName);
   }
 
   I.replaceAllUsesWith(Vec);
   I.eraseFromParent();
   return true;
 }
 
-bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
+bool AMDGPUCodeGenPrepareImpl::visitIntrinsicInst(IntrinsicInst &I) {
   switch (I.getIntrinsicID()) {
   case Intrinsic::bitreverse:
     return visitBitreverseIntrinsicInst(I);
   case Intrinsic::minnum:
     return visitMinNum(I);
   default:
     return false;
   }
 }
 
-bool AMDGPUCodeGenPrepare::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
+bool AMDGPUCodeGenPrepareImpl::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
   bool Changed = false;
 
   if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
       UA->isUniform(&I))
     Changed |= promoteUniformBitreverseToI32(I);
 
   return Changed;
 }
 
 /// Match non-nan fract pattern.
 ///   minnum(fsub(x, floor(x)), nextafter(1.0, -1.0)
 ///
 /// If fract is a useful instruction for the subtarget. Does not account for the
 /// nan handling; the instruction has a nan check on the input value.
-Value *AMDGPUCodeGenPrepare::matchFractPat(IntrinsicInst &I) {
+Value *AMDGPUCodeGenPrepareImpl::matchFractPat(IntrinsicInst &I) {
   if (ST->hasFractBug())
     return nullptr;
 
   if (I.getIntrinsicID() != Intrinsic::minnum)
     return nullptr;
 
   Type *Ty = I.getType();
   if (!isLegalFloatingTy(Ty->getScalarType()))
Value *AMDGPUCodeGenPrepareImpl::matchFractPat(IntrinsicInst &I) {
 
   Value *FloorSrc;
   if (match(Arg0, m_FSub(m_Value(FloorSrc),
                          m_Intrinsic<Intrinsic::floor>(m_Deferred(FloorSrc)))))
     return FloorSrc;
   return nullptr;
 }
 
-Value *AMDGPUCodeGenPrepare::applyFractPat(IRBuilder<> &Builder,
+Value *AMDGPUCodeGenPrepareImpl::applyFractPat(IRBuilder<> &Builder,
                                                Value *FractArg) {
   SmallVector<Value *, 4> FractVals;
   extractValues(Builder, FractVals, FractArg);
 
   SmallVector<Value *, 4> ResultVals(FractVals.size());
 
   Type *Ty = FractArg->getType()->getScalarType();
   for (unsigned I = 0, E = FractVals.size(); I != E; ++I) {
     ResultVals[I] =
         Builder.CreateIntrinsic(Intrinsic::amdgcn_fract, {Ty}, {FractVals[I]});
   }
 
   return insertValues(Builder, FractArg->getType(), ResultVals);
 }
 
-bool AMDGPUCodeGenPrepare::visitMinNum(IntrinsicInst &I) {
+bool AMDGPUCodeGenPrepareImpl::visitMinNum(IntrinsicInst &I) {
   Value *FractArg = matchFractPat(I);
   if (!FractArg)
     return false;
 
   // Match pattern for fract intrinsic in contexts where the nan check has been
   // optimized out (and hope the knowledge the source can't be nan wasn't lost).
   if (!I.hasNoNaNs() && !isKnownNeverNaN(FractArg, *DL, TLInfo))
     return false;
 
   IRBuilder<> Builder(&I);
   FastMathFlags FMF = I.getFastMathFlags();
   FMF.setNoNaNs();
   Builder.setFastMathFlags(FMF);
 
   Value *Fract = applyFractPat(Builder, FractArg);
   Fract->takeName(&I);
   I.replaceAllUsesWith(Fract);
 
   RecursivelyDeleteTriviallyDeadInstructions(&I, TLInfo);
   return true;
 }
 
 bool AMDGPUCodeGenPrepare::doInitialization(Module &M) {
-  Mod = &M;
-  DL = &Mod->getDataLayout();
+  Impl.Mod = &M;
+  Impl.DL = &Impl.Mod->getDataLayout();
   return false;
 }
 
 bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
   auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
   if (!TPC)
     return false;
 
   const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
-  ST = &TM.getSubtarget<GCNSubtarget>(F);
-  TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
-  UA = &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
-
+  Impl.TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+  Impl.ST = &TM.getSubtarget<GCNSubtarget>(F);
+  Impl.AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+  Impl.UA = &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
   auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
-  DT = DTWP ? &DTWP->getDomTree() : nullptr;
-
-  HasUnsafeFPMath = hasUnsafeFPMath(F);
-
+  Impl.DT = DTWP ? &DTWP->getDomTree() : nullptr;
+  Impl.HasUnsafeFPMath = hasUnsafeFPMath(F);
   SIModeRegisterDefaults Mode(F);
-  HasFP32Denormals = Mode.allFP32Denormals();
-
-  bool MadeChange = false;
-
-  Function::iterator NextBB;
-  for (Function::iterator FI = F.begin(), FE = F.end(); FI != FE; FI = NextBB) {
-    BasicBlock *BB = &*FI;
-    NextBB = std::next(FI);
-
-    BasicBlock::iterator Next;
-    for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; I = Next) {
-      Next = std::next(I);
-
-      MadeChange |= visit(*I);
-
-      if (Next != E) { // Control flow changed
-        BasicBlock *NextInstBB = Next->getParent();
-        if (NextInstBB != BB) {
-          BB = NextInstBB;
-          E = BB->end();
-          FE = F.end();
-        }
-      }
-    }
-  }
+  Impl.HasFP32Denormals = Mode.allFP32Denormals();
+  return Impl.run(F);
 
-  return MadeChange;
+PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
+                                                FunctionAnalysisManager &FAM) {
+  AMDGPUCodeGenPrepareImpl Impl;
+  Impl.Mod = F.getParent();
+  Impl.DL = &Impl.Mod->getDataLayout();
+  Impl.TLInfo = &FAM.getResult<TargetLibraryAnalysis>(F);
+  Impl.ST = &TM.getSubtarget<GCNSubtarget>(F);
+  Impl.AC = &FAM.getResult<AssumptionAnalysis>(F);
+  Impl.UA = &FAM.getResult<UniformityInfoAnalysis>(F);
+  Impl.DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
+  Impl.HasUnsafeFPMath = hasUnsafeFPMath(F);
+  SIModeRegisterDefaults Mode(F);
+  Impl.HasFP32Denormals = Mode.allFP32Denormals();
+  PreservedAnalyses PA = PreservedAnalyses::none();
+  if (!Impl.FlowChanged)
+    PA.preserveSet<CFGAnalyses>();
+  return Impl.run(F) ? PA : PreservedAnalyses::all();
 }
 
 INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
                       "AMDGPU IR optimizations", false, false)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
 INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
                     false, false)
 
 char AMDGPUCodeGenPrepare::ID = 0;
 
 FunctionPass *llvm::createAMDGPUCodeGenPreparePass() {
   return new AMDGPUCodeGenPrepare();
 }

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

PB.registerPipelineParsingCallback(
         if (PassName == "amdgpu-unify-divergent-exit-nodes") {
           PM.addPass(AMDGPUUnifyDivergentExitNodesPass());
           return true;
         }
         if (PassName == "amdgpu-atomic-optimizer") {
           PM.addPass(AMDGPUAtomicOptimizerPass(*this));
           return true;
         }
+        if (PassName == "amdgpu-codegenprepare") {
+          PM.addPass(AMDGPUCodeGenPreparePass(*this));
+          return true;
+        }
         return false;
       });
 
   PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) {
     FAM.registerPass([&] { return AMDGPUAA(); });
   });
 
   PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) {

llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-mul24.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -amdgpu-codegenprepare %s | FileCheck -check-prefix=SI %s
+; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -passes=amdgpu-codegenprepare %s | FileCheck -check-prefix=SI %s
 ; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare %s | FileCheck -check-prefix=VI %s
 ; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare-mul24=0 -amdgpu-codegenprepare %s | FileCheck -check-prefix=DISABLED %s
 
 define i16 @mul_i16(i16 %lhs, i16 %rhs) {
 ; SI-LABEL: @mul_i16(
 ; SI-NEXT:    [[TMP1:%.*]] = zext i16 [[LHS:%.*]] to i32
 ; SI-NEXT:    [[TMP2:%.*]] = zext i16 [[RHS:%.*]] to i32
 ; SI-NEXT:    [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24(i32 [[TMP1]], i32 [[TMP2]])