[InstCombine][Assumes] Cannonicalize assumes in instcombine

[Tyker]

Summary

this allows remving assumes about allocas, remove bitcasts or GEP used only in assume

[Tyker]

this was a mistake i intended to update D82703

llvm/include/llvm/Analysis/AssumeBundleQueries.h

struct RetainedKnowledge {
   Attribute::AttrKind AttrKind = Attribute::None;
   unsigned ArgValue = 0;
   Value *WasOn = nullptr;
   bool operator==(RetainedKnowledge Other) const {
     return AttrKind == Other.AttrKind && WasOn == Other.WasOn &&
            ArgValue == Other.ArgValue;
   }
   bool operator!=(RetainedKnowledge Other) const { return !(*this == Other); }
+  /// This is only intended for use in std::min/std::max between attribute that
+  /// only differ in ArgValue.
+  bool operator<(RetainedKnowledge Other) const {
+    assert(AttrKind == Other.AttrKind && WasOn == Other.WasOn &&
+           "This is only intend for use in min/max to select the best for "
+           "RetainedKnowledge that is otherwise equal");
+    return ArgValue < Other.ArgValue;
+  }
   operator bool() const { return AttrKind != Attribute::None; }
   static RetainedKnowledge none() { return RetainedKnowledge{}; }
 };
 
 /// Retreive the information help by Assume on the operand at index Idx.
 /// Assume should be an llvm.assume and Idx should be in the operand bundle.
 RetainedKnowledge getKnowledgeFromOperandInAssume(CallInst &Assume,
                                                   unsigned Idx);

llvm/include/llvm/Analysis/ValueTracking.h

constexpr unsigned MaxAnalysisRecursionDepth = 6;
   /// Return true if it is an intrinsic that cannot be speculated but also
   /// cannot trap.
   bool isAssumeLikeIntrinsic(const Instruction *I);
 
   /// Return true if it is valid to use the assumptions provided by an
   /// assume intrinsic, I, at the point in the control-flow identified by the
   /// context instruction, CxtI.
   bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI,
-                               const DominatorTree *DT = nullptr);
+                               const DominatorTree *DT = nullptr,
+                               bool AllowEphemeral = false);
 
   enum class OverflowResult {
     /// Always overflows in the direction of signed/unsigned min value.
     AlwaysOverflowsLow,
     /// Always overflows in the direction of signed/unsigned max value.
     AlwaysOverflowsHigh,
     /// May or may not overflow.
     MayOverflow,

llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h

 // transformation may destroy or worsen information that can be derived from the
 // IR.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H
 #define LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H
 
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
 class IntrinsicInst;
 class AssumptionCache;
 class DominatorTree;

 /// contained in the instruction I.
 /// The AssumptionCache must be provided if it is available or the cache may
 /// become silently be invalid.
 /// The DominatorTree can optionally be provided to enable cross-block
 /// reasoning.
 void salvageKnowledge(Instruction *I, AssumptionCache *AC = nullptr,
                       DominatorTree *DT = nullptr);
 
+/// Build and return a new assume created from the provided knowledge
+/// if the knowledge in the assume is fully redundant this will return nullptr
+IntrinsicInst *buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
+                                        Instruction *CtxI,
+                                        AssumptionCache *AC = nullptr,
+                                        DominatorTree *DT = nullptr);
+
 /// This pass attempts to minimize the number of assume without loosing any
 /// information.
 struct AssumeSimplifyPass : public PassInfoMixin<AssumeSimplifyPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
 FunctionPass *createAssumeSimplifyPass();
 
 /// This pass will try to build an llvm.assume for every instruction in the
 /// function. Its main purpose is testing.
 struct AssumeBuilderPass : public PassInfoMixin<AssumeBuilderPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
+/// return a canonical RetainedKnowledge from RK, Assume holding RK,
+/// Will return a none RetainedKnowledge if RK is useless.
+RetainedKnowledge simplifyRetainedKnowledge(CallBase *Assume,
+                                            RetainedKnowledge RK,
+                                            AssumptionCache *AC,
+                                            DominatorTree *DT);
+
 } // namespace llvm
 
 #endif

llvm/lib/Analysis/AssumeBundleQueries.cpp

if (RetainedKnowledge RK =
       }
   }
   return RetainedKnowledge::none();
 }
 
 RetainedKnowledge llvm::getKnowledgeValidInContext(
     const Value *V, ArrayRef<Attribute::AttrKind> AttrKinds,
     const Instruction *CtxI, const DominatorTree *DT, AssumptionCache *AC) {
-  return getKnowledgeForValue(V, AttrKinds, AC,
-                              [&](auto, Instruction *I, auto) {
-                                return isValidAssumeForContext(I, CtxI, DT);
-                              });
+  return getKnowledgeForValue(
+      V, AttrKinds, AC, [&](auto, Instruction *I, auto) {
+        return isValidAssumeForContext(I, CtxI, DT, /*AllowEphemeral*/ true);
 }

llvm/lib/Analysis/Loads.cpp

 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Statepoint.h"
if (!CheckForNonNull || isKnownNonZero(V, DL, 0, nullptr, CtxI, DT)) {
       // that each step advanced by a multiple of the alignment. If our base is
       // properly aligned, then the original offset accessed must also be.
       Type *Ty = V->getType();
       assert(Ty->isSized() && "must be sized");
       APInt Offset(DL.getTypeStoreSizeInBits(Ty), 0);
       return isAligned(V, Offset, Alignment, DL);
     }
 
+  /// Look through assumes to see if both dereferencability and alignment can be
+  /// provent by an assume
+  RetainedKnowledge AlignRK;
+  RetainedKnowledge DerefRK;
+  if (getKnowledgeForValue(
+          V, {Attribute::Dereferenceable, Attribute::Alignment}, nullptr,
+          [&](RetainedKnowledge RK, Instruction *Assume, auto) {
+            if (!isValidAssumeForContext(Assume, CtxI))
+              return false;
+            if (RK.AttrKind == Attribute::Alignment)
+              AlignRK = std::max(AlignRK, RK);
+            if (RK.AttrKind == Attribute::Dereferenceable)
+              DerefRK = std::max(DerefRK, RK);
+            if (AlignRK && DerefRK && AlignRK.ArgValue >= Alignment.value() &&
+                DerefRK.ArgValue >= Size.getZExtValue())
+              return true; // We have found what we needed so here
+            return false;  // Other assumes may have better information. so keep
+                           // looking
+          }))
+    return true;
+  /// TODO refactor this function to be able to search independently for
+  /// Dereferencability and Alignment requirements.
+
   // For GEPs, determine if the indexing lands within the allocated object.
   if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
     const Value *Base = GEP->getPointerOperand();
 
     APInt Offset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
     if (!GEP->accumulateConstantOffset(DL, Offset) || Offset.isNegative() ||
         !Offset.urem(APInt(Offset.getBitWidth(), Alignment.value()))
              .isMinValue())

llvm/lib/Analysis/ValueTracking.cpp

if (Function *F = CI->getCalledFunction())
         return true;
       }
 
   return false;
 }
 
 bool llvm::isValidAssumeForContext(const Instruction *Inv,
                                    const Instruction *CxtI,
-                                   const DominatorTree *DT) {
+                                   const DominatorTree *DT,
+                                   bool AllowEphemeral) {
   // There are two restrictions on the use of an assume:
   //  1. The assume must dominate the context (or the control flow must
   //     reach the assume whenever it reaches the context).
   //  2. The context must not be in the assume's set of ephemeral values
   //     (otherwise we will use the assume to prove that the condition
   //     feeding the assume is trivially true, thus causing the removal of
   //     the assume).
 
if (Inv->getParent() == CxtI->getParent()) {
 
     // The context comes first, but they're both in the same block.
     // Make sure there is nothing in between that might interrupt
     // the control flow, not even CxtI itself.
     for (BasicBlock::const_iterator I(CxtI), IE(Inv); I != IE; ++I)
       if (!isGuaranteedToTransferExecutionToSuccessor(&*I))
         return false;
 
-    return !isEphemeralValueOf(Inv, CxtI);
+    return AllowEphemeral || !isEphemeralValueOf(Inv, CxtI);
   }
 
   // Inv and CxtI are in different blocks.
   if (DT) {
     if (DT->dominates(Inv, CxtI))
       return true;
   } else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor()) {
     // We don't have a DT, but this trivially dominates.
static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known,
                                        unsigned Depth, const Query &Q) {
   // Use of assumptions is context-sensitive. If we don't have a context, we
   // cannot use them!
   if (!Q.AC || !Q.CxtI)
     return;
 
   unsigned BitWidth = Known.getBitWidth();
 
+  if (V->getType()->isPointerTy()) {
+    if (getKnowledgeForValue(
+            V, {Attribute::Alignment}, Q.AC,
+            [&](RetainedKnowledge RK, Instruction *Assume, auto) {
+              if (isValidAssumeForContext(Assume, Q.CxtI, Q.DT,
+                                          /*AllowEphemeral*/ true))
+                Known.Zero.setLowBits(countTrailingZeros(RK.ArgValue));
+              return false; /// There may be a better assume later so keep going
+            }))
+      return;
+  }
+
   // Note that the patterns below need to be kept in sync with the code
   // in AssumptionCache::updateAffectedValues.
 
   for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
     if (!AssumeVH)
       continue;
     CallInst *I = cast<CallInst>(AssumeVH);
     assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <cstring>
 #include <utility>
 #include <vector>
 
 using namespace llvm;
 using namespace PatternMatch;
 
 #define DEBUG_TYPE "instcombine"
 
 STATISTIC(NumSimplified, "Number of library calls simplified");
 
 static cl::opt<unsigned> GuardWideningWindow(
     "instcombine-guard-widening-window",
     cl::init(3),
     cl::desc("How wide an instruction window to bypass looking for "
              "another guard"));
 
+/// enable preservation of attributes in assume like:
+/// call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]
+extern cl::opt<bool> EnableKnowledgeRetention;
+
 /// Return the specified type promoted as it would be to pass though a va_arg
 /// area.
 static Type *getPromotedType(Type *Ty) {
   if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
     if (ITy->getBitWidth() < 32)
       return Type::getInt32Ty(Ty->getContext());
   }
   return Ty;
if (removeTriviallyEmptyRange(*II, *this, [](const IntrinsicInst &I) {
           return I.getIntrinsicID() == Intrinsic::lifetime_start;
         }))
       return nullptr;
     break;
   case Intrinsic::assume: {
     Value *IIOperand = II->getArgOperand(0);
     SmallVector<OperandBundleDef, 4> OpBundles;
     II->getOperandBundlesAsDefs(OpBundles);
-    bool HasOpBundles = !OpBundles.empty();
+
+    /// This will remove the boolean Condition from the assume given as
+    /// arguement and remove the assume if it becomes useless.
+    /// always returns nullptr for use as a return values.
+    auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * {
+      assert(isa<IntrinsicInst>(Assume));
+      if (isAssumeWithEmptyBundle(*cast<IntrinsicInst>(II)))
+        return eraseInstFromFunction(CI);
+      replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext()));
+      return nullptr;
+    };
     // Remove an assume if it is followed by an identical assume.
     // TODO: Do we need this? Unless there are conflicting assumptions, the
     // computeKnownBits(IIOperand) below here eliminates redundant assumes.
     Instruction *Next = II->getNextNonDebugInstruction();
-    if (HasOpBundles &&
-        match(Next, m_Intrinsic<Intrinsic::assume>(m_Specific(IIOperand))) &&
-        !cast<IntrinsicInst>(Next)->hasOperandBundles())
-      return eraseInstFromFunction(CI);
+    if (match(Next, m_Intrinsic<Intrinsic::assume>(m_Specific(IIOperand))))
+      return RemoveConditionFromAssume(Next);
 
     // Canonicalize assume(a && b) -> assume(a); assume(b);
     // Note: New assumption intrinsics created here are registered by
     // the InstCombineIRInserter object.
     FunctionType *AssumeIntrinsicTy = II->getFunctionType();
     Value *AssumeIntrinsic = II->getCalledOperand();
     Value *A, *B;
     if (match(IIOperand, m_And(m_Value(A), m_Value(B)))) {
       Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, OpBundles,
                          II->getName());
       Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());
       return eraseInstFromFunction(*II);
     }
     // assume(!(a || b)) -> assume(!a); assume(!b);
     if (match(IIOperand, m_Not(m_Or(m_Value(A), m_Value(B))))) {
       Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
                          Builder.CreateNot(A), OpBundles, II->getName());
       Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
                          Builder.CreateNot(B), II->getName());
       return eraseInstFromFunction(*II);
     }
 
+    // Convert nonnull assume like:
+    // %A = icmp ne i32* %PTR, null
+    // call void @llvm.assume(i1 %A)
+    // into
+    // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]
+    CmpInst::Predicate Pred;
+    if (EnableKnowledgeRetention &&
+        match(IIOperand, m_Cmp(Pred, m_Value(A), m_Zero())) &&
+        Pred == CmpInst::ICMP_NE && A->getType()->isPointerTy()) {
+      if (IntrinsicInst *Replacement = buildAssumeFromKnowledge(
+              {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) {
+
+        Replacement->insertBefore(Next);
+        AC.registerAssumption(Replacement);
+      }
+      return RemoveConditionFromAssume(II);
+    }
+
+    // Convert alignment assume like:
+    // %A = icmp ne i32* %PTR, null
+    // %B = ptrtoint i32* %A to i64
+    // %C = and i64 %B, Constant
+    // %D = icmp eq i64 %C, 0
+    // call void @llvm.assume(i1 %D)
+    // into
+    // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64  Constant + 1)]
+    uint64_t AlignMask;
+    if (EnableKnowledgeRetention &&
+        match(IIOperand,
+              m_Cmp(Pred, m_And(m_Value(A), m_ConstantInt(AlignMask)),
+                    m_Zero())) &&
+        Pred == CmpInst::ICMP_EQ) {
+      if (isPowerOf2_64(AlignMask + 1)) {
+        uint64_t Offset = 0;
+        match(A, m_Add(m_Value(A), m_ConstantInt(Offset)));
+        if (match(A, m_PtrToInt(m_Value(A)))) {
+          /// Note: this doesn't preserve the offset information but merges
+          /// offset and alignment.
+          /// TODO: we can generate a GEP instead of merging the alignment with
+          /// the offset.
+          RetainedKnowledge RK{Attribute::Alignment,
+                               (unsigned)MinAlign(Offset, AlignMask + 1), A};
+          if (IntrinsicInst *Replacement =
+                  buildAssumeFromKnowledge(RK, Next, &AC, &DT)) {
+
+            Replacement->insertAfter(II);
+            AC.registerAssumption(Replacement);
+          }
+          return RemoveConditionFromAssume(II);
+        }
+      }
+    }
+
+    /// Canonicalize Knowledge help in operand bundles.
+    if (II->hasOperandBundles()) {
+      for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
+        auto &BOI = II->bundle_op_info_begin()[Idx];
+        RetainedKnowledge RK = llvm::getKnowledgeFromBundle(*II, BOI);
+        if (BOI.End - BOI.Begin > 2)
+          continue; // Prevent reducing knowledge in an align with offset since
+                    // extracting a RetainedKnowledge form then looses
+        RetainedKnowledge CanonRK = llvm::simplifyRetainedKnowledge(
+            II, RK, &getAssumptionCache(), &getDominatorTree());
+        if (CanonRK != RK) {
+          if (!CanonRK) {
+            if (BOI.End - BOI.Begin > 0)
+              Value::dropDroppableUse(II->op_begin()[BOI.Begin]);
+            return nullptr;
+          }
+          assert(RK.AttrKind == CanonRK.AttrKind);
+          if (BOI.End - BOI.Begin > 0)
+            II->op_begin()[BOI.Begin].set(CanonRK.WasOn);
+          if (BOI.End - BOI.Begin > 1)
+            II->op_begin()[BOI.Begin + 1].set(ConstantInt::get(
+                Type::getInt64Ty(II->getContext()), CanonRK.ArgValue));
+          if (RK.WasOn)
+            Worklist.pushValue(RK.WasOn);
+          return II;
+        }
+      }
+    }
+
     // assume( (load addr) != null ) -> add 'nonnull' metadata to load
     // (if assume is valid at the load)
-    CmpInst::Predicate Pred;
     Instruction *LHS;
     if (match(IIOperand, m_ICmp(Pred, m_Instruction(LHS), m_Zero())) &&
         Pred == ICmpInst::ICMP_NE && LHS->getOpcode() == Instruction::Load &&
         LHS->getType()->isPointerTy() &&
         isValidAssumeForContext(II, LHS, &DT)) {
       MDNode *MD = MDNode::get(II->getContext(), None);
       LHS->setMetadata(LLVMContext::MD_nonnull, MD);
-      if (!HasOpBundles)
-        return eraseInstFromFunction(*II);
+      return RemoveConditionFromAssume(II);
 
       // TODO: apply nonnull return attributes to calls and invokes
       // TODO: apply range metadata for range check patterns?
     }
 
     // If there is a dominating assume with the same condition as this one,
     // then this one is redundant, and should be removed.
     KnownBits Known(1);

llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp

 //===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "assume-builder"
-
 #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Dominators.h"
cl::opt<bool> ShouldPreserveAllAttributes(
     cl::desc("enable preservation of all attrbitues. even those that are "
              "unlikely to be usefull"));
 
 cl::opt<bool> EnableKnowledgeRetention(
     "enable-knowledge-retention", cl::init(false), cl::Hidden,
     cl::desc(
         "enable preservation of attributes throughout code transformation"));
 
+#define DEBUG_TYPE "assume-builder"
+
 STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder");
 STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built");
 STATISTIC(NumAssumesMerged,
           "Number of assume merged by the assume simplify pass");
 STATISTIC(NumAssumesRemoved,
           "Number of assume removed by the assume simplify pass");
 
 DEBUG_COUNTER(BuildAssumeCounter, "assume-builder-counter",
case Attribute::Cold:
       return true;
     default:
       return false;
   }
 }
 
 /// This function will try to transform the given knowledge into a more
 /// canonical one. the canonical knowledge maybe the given one.
-RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, Module *M) {
+RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, DataLayout DL) {
   switch (RK.AttrKind) {
   default:
     return RK;
   case Attribute::NonNull:
     RK.WasOn = getUnderlyingObject(RK.WasOn);
     return RK;
   case Attribute::Alignment: {
     Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
       if (auto *GEP = dyn_cast<GEPOperator>(Strip))
         RK.ArgValue =
-            MinAlign(RK.ArgValue,
-                     GEP->getMaxPreservedAlignment(M->getDataLayout()).value());
+            MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value());
     });
     RK.WasOn = V;
     return RK;
   }
   case Attribute::Dereferenceable:
   case Attribute::DereferenceableOrNull: {
     int64_t Offset = 0;
-    Value *V = GetPointerBaseWithConstantOffset(
-        RK.WasOn, Offset, M->getDataLayout(), /*AllowNonInBounds*/ false);
+    Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL,
+                                                /*AllowNonInBounds*/ false);
     if (Offset < 0)
       return RK;
     RK.ArgValue = RK.ArgValue + Offset;
     RK.WasOn = V;
   }
   }
   return RK;
 }
 
 /// This class contain all knowledge that have been gather while building an
 /// llvm.assume and the function to manipulate it.
 struct AssumeBuilderState {
   Module *M;
 
   using MapKey = std::pair<Value *, Attribute::AttrKind>;
   SmallMapVector<MapKey, unsigned, 8> AssumedKnowledgeMap;
-  Instruction *InstBeingRemoved = nullptr;
+  Instruction *InstBeingModified = nullptr;
   AssumptionCache* AC = nullptr;
   DominatorTree* DT = nullptr;
 
   AssumeBuilderState(Module *M, Instruction *I = nullptr,
                      AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr)
-      : M(M), InstBeingRemoved(I), AC(AC), DT(DT) {}
+      : M(M), InstBeingModified(I), AC(AC), DT(DT) {}
 
   bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) {
-    if (!InstBeingRemoved || !RK.WasOn)
+    if (!InstBeingModified || !RK.WasOn)
       return false;
     bool HasBeenPreserved = false;
     Use* ToUpdate = nullptr;
     getKnowledgeForValue(
         RK.WasOn, {RK.AttrKind}, AC,
         [&](RetainedKnowledge RKOther, Instruction *Assume,
             const CallInst::BundleOpInfo *Bundle) {
-          if (!isValidAssumeForContext(Assume, InstBeingRemoved, DT))
+          if (!isValidAssumeForContext(Assume, InstBeingModified, DT))
             return false;
           if (RKOther.ArgValue >= RK.ArgValue) {
             HasBeenPreserved = true;
             return true;
-          } else if (isValidAssumeForContext(InstBeingRemoved, Assume,
+          } else if (isValidAssumeForContext(InstBeingModified, Assume,
                                              DT)) {
             HasBeenPreserved = true;
             IntrinsicInst *Intr = cast<IntrinsicInst>(Assume);
             ToUpdate = &Intr->op_begin()[Bundle->Begin + ABA_Argument];
             return true;
           }
           return false;
         });
if (auto *Arg = dyn_cast<Argument>(RK.WasOn)) {
         return false;
       return true;
     }
     if (auto *Inst = dyn_cast<Instruction>(RK.WasOn))
       if (wouldInstructionBeTriviallyDead(Inst)) {
         if (RK.WasOn->use_empty())
           return false;
         Use *SingleUse = RK.WasOn->getSingleUndroppableUse();
-        if (SingleUse && SingleUse->getUser() == InstBeingRemoved)
+        if (SingleUse && SingleUse->getUser() == InstBeingModified)
           return false;
       }
     return true;
   }
 
   void addKnowledge(RetainedKnowledge RK) {
-    RK = canonicalizedKnowledge(RK, M);
+    RK = canonicalizedKnowledge(RK, M->getDataLayout());
 
     if (!isKnowledgeWorthPreserving(RK))
       return;
 
     if (tryToPreserveWithoutAddingAssume(RK))
       return;
     MapKey Key{RK.WasOn, RK.AttrKind};
     auto Lookup = AssumedKnowledgeMap.find(Key);
void llvm::salvageKnowledge(Instruction *I, AssumptionCache *AC,
   Builder.addInstruction(I);
   if (IntrinsicInst *Intr = Builder.build()) {
     Intr->insertBefore(I);
     if (AC)
       AC->registerAssumption(Intr);
   }
 }
 
+IntrinsicInst *
+llvm::buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
+                               Instruction *CtxI, AssumptionCache *AC,
+                               DominatorTree *DT) {
+  AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT);
+  for (const RetainedKnowledge &RK : Knowledge)
+    Builder.addKnowledge(RK);
+  return Builder.build();
+}
+
+RetainedKnowledge llvm::simplifyRetainedKnowledge(CallBase *Assume,
+                                                  RetainedKnowledge RK,
+                                                  AssumptionCache *AC,
+                                                  DominatorTree *DT) {
+  if (!Assume)
+    return canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout());
+  AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT);
+  RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout());
+
+  if (!Builder.isKnowledgeWorthPreserving(RK))
+    return RetainedKnowledge::none();
+
+  if (Builder.tryToPreserveWithoutAddingAssume(RK))
+    return RetainedKnowledge::none();
+  return RK;
+}
+
 namespace {
 
 struct AssumeSimplify {
   Function &F;
   AssumptionCache &AC;
   DominatorTree *DT;
   LLVMContext &C;
   SmallDenseSet<IntrinsicInst *> CleanupToDo;

llvm/test/Analysis/BasicAA/featuretest.ll

 ; operations on another array.  Important for scientific codes.
 ;
 define i32 @different_array_test(i64 %A, i64 %B) {
 ; CHECK-LABEL: @different_array_test(
 ; CHECK-NEXT:    [[ARRAY11:%.*]] = alloca [100 x i32], align 4
 ; CHECK-NEXT:    [[ARRAY22:%.*]] = alloca [200 x i32], align 4
 ; CHECK-NEXT:    [[ARRAY22_SUB:%.*]] = getelementptr inbounds [200 x i32], [200 x i32]* [[ARRAY22]], i64 0, i64 0
 ; CHECK-NEXT:    [[ARRAY11_SUB:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* [[ARRAY11]], i64 0, i64 0
-; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[ARRAY11_SUB]], i32 4) ]
 ; CHECK-NEXT:    call void @external(i32* nonnull [[ARRAY11_SUB]])
 ; CHECK-NEXT:    call void @external(i32* nonnull [[ARRAY22_SUB]])
 ; CHECK-NEXT:    [[POINTER2:%.*]] = getelementptr [200 x i32], [200 x i32]* [[ARRAY22]], i64 0, i64 [[B:%.*]]
 ; CHECK-NEXT:    store i32 7, i32* [[POINTER2]], align 4
 ; CHECK-NEXT:    ret i32 0
 ;
   %Array1 = alloca i32, i32 100
   %Array2 = alloca i32, i32 200

 ; NO_ASSUME-NEXT:    [[C1:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 1
 ; NO_ASSUME-NEXT:    [[C:%.*]] = bitcast i32* [[C1]] to i8*
 ; NO_ASSUME-NEXT:    store i8 42, i8* [[C]], align 1
 ; NO_ASSUME-NEXT:    ret i32 0
 ;
 ; USE_ASSUME-LABEL: @gep_distance_test3(
 ; USE_ASSUME-NEXT:    [[C1:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 1
 ; USE_ASSUME-NEXT:    [[C:%.*]] = bitcast i32* [[C1]] to i8*
-; USE_ASSUME-NEXT:    store i8 42, i8* [[C]], align 1
+; USE_ASSUME-NEXT:    store i8 42, i8* [[C]], align 4
 ; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[A]], i64 4), "nonnull"(i32* [[A]]), "align"(i32* [[A]], i64 4) ]
 ; USE_ASSUME-NEXT:    ret i32 0
 ;
   %X = load i32, i32* %A
   %B = bitcast i32* %A to i8*
   %C = getelementptr i8, i8* %B, i64 4
   store i8 42, i8* %C
   %Y = load i32, i32* %A

llvm/test/Analysis/ValueTracking/assume-queries-counter.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; REQUIRES: asserts
 
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=0,assume-queries-counter-count=1 -S | FileCheck %s --check-prefixes=SAME,COUNTER1
 ; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=1,assume-queries-counter-count=2 -S | FileCheck %s --check-prefixes=SAME,COUNTER2
-; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=2,assume-queries-counter-count=1 -S | FileCheck %s --check-prefixes=SAME,COUNTER3
+; RUN: opt < %s -instcombine --debug-counter=assume-queries-counter-skip=2,assume-queries-counter-count=5 -S | FileCheck %s --check-prefixes=SAME,COUNTER3
 
 declare i1 @get_val()
 declare void @llvm.assume(i1)
 
 define dso_local i1 @test1(i32* readonly %0) {
 ; COUNTER1-LABEL: @test1(
 ; COUNTER1-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ]
-; COUNTER1-NEXT:    ret i1 false
+; COUNTER1-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null
+; COUNTER1-NEXT:    ret i1 [[TMP2]]
 ;
 ; COUNTER2-LABEL: @test1(
 ; COUNTER2-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ]
-; COUNTER2-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null
-; COUNTER2-NEXT:    ret i1 [[TMP2]]
+; COUNTER2-NEXT:    ret i1 false
 ;
 ; COUNTER3-LABEL: @test1(
 ; COUNTER3-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ]
 ; COUNTER3-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null
 ; COUNTER3-NEXT:    ret i1 [[TMP2]]
 ;
   call void @llvm.assume(i1 true) ["nonnull"(i32* %0)]
   %2 = icmp eq i32* %0, null
   ret i1 %2
 }
 
 define dso_local i1 @test2(i32* readonly %0) {
 ; COUNTER1-LABEL: @test2(
 ; COUNTER1-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0:%.*]], null
 ; COUNTER1-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0]]) ]
 ; COUNTER1-NEXT:    ret i1 [[TMP2]]
 ;
 ; COUNTER2-LABEL: @test2(
-; COUNTER2-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ]
-; COUNTER2-NEXT:    ret i1 false
+; COUNTER2-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0:%.*]], null
+; COUNTER2-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0]]) ]
+; COUNTER2-NEXT:    ret i1 [[TMP2]]
 ;
 ; COUNTER3-LABEL: @test2(
-; COUNTER3-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0:%.*]], null
-; COUNTER3-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0]]) ]
-; COUNTER3-NEXT:    ret i1 [[TMP2]]
+; COUNTER3-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[TMP0:%.*]]) ]
+; COUNTER3-NEXT:    ret i1 false
 ;
   %2 = icmp eq i32* %0, null
   call void @llvm.assume(i1 true) ["nonnull"(i32* %0)]
   ret i1 %2
 }
 
 define dso_local i32 @test4(i32* readonly %0, i1 %cond) {
-; COUNTER1-LABEL: @test4(
-; COUNTER1-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ]
-; COUNTER1-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
-; COUNTER1:       B:
-; COUNTER1-NEXT:    br label [[A]]
-; COUNTER1:       A:
-; COUNTER1-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null
-; COUNTER1-NEXT:    br i1 [[TMP2]], label [[TMP5:%.*]], label [[TMP3:%.*]]
-; COUNTER1:       3:
-; COUNTER1-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP0]], align 4
-; COUNTER1-NEXT:    br label [[TMP5]]
-; COUNTER1:       5:
-; COUNTER1-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP4]], [[TMP3]] ], [ 0, [[A]] ]
-; COUNTER1-NEXT:    ret i32 [[TMP6]]
-;
-; COUNTER2-LABEL: @test4(
-; COUNTER2-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ]
-; COUNTER2-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
-; COUNTER2:       B:
-; COUNTER2-NEXT:    br label [[A]]
-; COUNTER2:       A:
-; COUNTER2-NEXT:    br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]]
-; COUNTER2:       2:
-; COUNTER2-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4
-; COUNTER2-NEXT:    br label [[TMP4]]
-; COUNTER2:       4:
-; COUNTER2-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ]
-; COUNTER2-NEXT:    ret i32 [[TMP5]]
-;
-; COUNTER3-LABEL: @test4(
-; COUNTER3-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ]
-; COUNTER3-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
-; COUNTER3:       B:
-; COUNTER3-NEXT:    br label [[A]]
-; COUNTER3:       A:
-; COUNTER3-NEXT:    br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]]
-; COUNTER3:       2:
-; COUNTER3-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4
-; COUNTER3-NEXT:    br label [[TMP4]]
-; COUNTER3:       4:
-; COUNTER3-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ]
-; COUNTER3-NEXT:    ret i32 [[TMP5]]
+; SAME-LABEL: @test4(
+; SAME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ]
+; SAME-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; SAME:       B:
+; SAME-NEXT:    br label [[A]]
+; SAME:       A:
+; SAME-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[TMP0]], null
+; SAME-NEXT:    br i1 [[TMP2]], label [[TMP5:%.*]], label [[TMP3:%.*]]
+; SAME:       3:
+; SAME-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP0]], align 4
+; SAME-NEXT:    br label [[TMP5]]
+; SAME:       5:
+; SAME-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP4]], [[TMP3]] ], [ 0, [[A]] ]
+; SAME-NEXT:    ret i32 [[TMP6]]
 ;
   call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4)]
   br i1 %cond, label %A, label %B
 
 B:
   br label %A
 
 A:

llvm/test/Analysis/ValueTracking/assume.ll

 ;
   %2 = icmp eq i32* %0, null
   call void @llvm.assume(i1 true) ["nonnull"(i32* %0)]
   ret i1 %2
 }
 
 define dso_local i32 @test4(i32* readonly %0, i1 %cond) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[TMP0:%.*]], i32 4), "align"(i32* [[TMP0]], i32 8) ]
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
 ; CHECK:       B:
 ; CHECK-NEXT:    br label [[A]]
 ; CHECK:       A:
 ; CHECK-NEXT:    br i1 false, label [[TMP4:%.*]], label [[TMP2:%.*]]
 ; CHECK:       2:
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP0]], align 8
 ; CHECK-NEXT:    br label [[TMP4]]
 ; CHECK:       4:
 ; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP3]], [[TMP2]] ], [ 0, [[A]] ]
 ; CHECK-NEXT:    ret i32 [[TMP5]]
 ;
-  call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4)]
+  call void @llvm.assume(i1 true) ["dereferenceable"(i32* %0, i32 4), "align"(i32* %0, i32 8)]
   br i1 %cond, label %A, label %B
 
 B:
   br label %A
 
 A:
   %2 = icmp eq i32* %0, null
   br i1 %2, label %5, label %3

llvm/test/Transforms/InstCombine/assume.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S  -instcombine-infinite-loop-threshold=2 | FileCheck --check-prefixes=SAME,DEFAULT %s
 ; RUN: opt < %s -instcombine --enable-knowledge-retention -S  -instcombine-infinite-loop-threshold=2 | FileCheck --check-prefixes=SAME,BUNDLES %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 declare void @llvm.assume(i1) #1
 
 ; Check that the alignment has been upgraded and that the assume has not
 ; been removed:
 
 define i32 @foo1(i32* %a) #0 {
-; SAME-LABEL: @foo1(
-; SAME-NEXT:    [[T0:%.*]] = load i32, i32* [[A:%.*]], align 32
-; SAME-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
-; SAME-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
-; SAME-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
-; SAME-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
-; SAME-NEXT:    ret i32 [[T0]]
+; DEFAULT-LABEL: @foo1(
+; DEFAULT-NEXT:    [[T0:%.*]] = load i32, i32* [[A:%.*]], align 32
+; DEFAULT-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A]] to i64
+; DEFAULT-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; DEFAULT-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; DEFAULT-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; DEFAULT-NEXT:    ret i32 [[T0]]
+;
+; BUNDLES-LABEL: @foo1(
+; BUNDLES-NEXT:    [[T0:%.*]] = load i32, i32* [[A:%.*]], align 32
+; BUNDLES-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 32) ]
+; BUNDLES-NEXT:    ret i32 [[T0]]
 ;
   %t0 = load i32, i32* %a, align 4
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
   %maskcond = icmp eq i64 %maskedptr, 0
   tail call void @llvm.assume(i1 %maskcond)
   ret i32 %t0
 }
 
 ; Same check as in @foo1, but make sure it works if the assume is first too.
 
 define i32 @foo2(i32* %a) #0 {
-; SAME-LABEL: @foo2(
-; SAME-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A:%.*]] to i64
-; SAME-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
-; SAME-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
-; SAME-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
-; SAME-NEXT:    [[T0:%.*]] = load i32, i32* [[A]], align 32
-; SAME-NEXT:    ret i32 [[T0]]
+; DEFAULT-LABEL: @foo2(
+; DEFAULT-NEXT:    [[PTRINT:%.*]] = ptrtoint i32* [[A:%.*]] to i64
+; DEFAULT-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; DEFAULT-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; DEFAULT-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; DEFAULT-NEXT:    [[T0:%.*]] = load i32, i32* [[A]], align 32
+; DEFAULT-NEXT:    ret i32 [[T0]]
+;
+; BUNDLES-LABEL: @foo2(
+; BUNDLES-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[A:%.*]], i64 32) ]
+; BUNDLES-NEXT:    [[T0:%.*]] = load i32, i32* [[A]], align 32
+; BUNDLES-NEXT:    ret i32 [[T0]]
 ;
   %ptrint = ptrtoint i32* %a to i64
   %maskedptr = and i64 %ptrint, 31
   %maskcond = icmp eq i64 %maskedptr, 0
   tail call void @llvm.assume(i1 %maskcond)
   %t0 = load i32, i32* %a, align 4
   ret i32 %t0
 }

 ; SAME-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P]], align 4
 ; SAME-NEXT:    ret i32 [[LOAD]]
 ;
   tail call void @llvm.assume(i1 true) ["nonnull"(i32* %P)]
   %load = load i32, i32* %P
   ret i32 %load
 }
 
+define i32 @bundle1A(i32* %P) {
+; SAME-LABEL: @bundle1A(
+; SAME-NEXT:    tail call void @llvm.assume(i1 true) [ "align"(i32* [[P:%.*]], i32 64) ]
+; SAME-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P]], align 64
+; SAME-NEXT:    ret i32 [[LOAD]]
+;
+  tail call void @llvm.assume(i1 true) ["align"(i32* %P, i32 64)]
+  %load = load i32, i32* %P
+  ret i32 %load
+}
+
 define i32 @bundle2(i32* %P) {
 ; SAME-LABEL: @bundle2(
 ; SAME-NEXT:    [[LOAD:%.*]] = load i32, i32* [[P:%.*]], align 4
 ; SAME-NEXT:    ret i32 [[LOAD]]
 ;
   tail call void @llvm.assume(i1 true) ["ignore"(i32* undef)]
   %load = load i32, i32* %P
   ret i32 %load
 }
 
 define i1 @nonnull1(i32** %a) {
-; SAME-LABEL: @nonnull1(
-; SAME-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8, !nonnull !6
-; SAME-NEXT:    tail call void @escape(i32* nonnull [[LOAD]])
-; SAME-NEXT:    ret i1 false
+; DEFAULT-LABEL: @nonnull1(
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8, !nonnull !6
+; DEFAULT-NEXT:    tail call void @escape(i32* nonnull [[LOAD]])
+; DEFAULT-NEXT:    ret i1 false
+;
+; BUNDLES-LABEL: @nonnull1(
+; BUNDLES-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; BUNDLES-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]) ]
+; BUNDLES-NEXT:    tail call void @escape(i32* nonnull [[LOAD]])
+; BUNDLES-NEXT:    ret i1 false
 ;
   %load = load i32*, i32** %a
   %cmp = icmp ne i32* %load, null
   tail call void @llvm.assume(i1 %cmp)
   tail call void @escape(i32* %load)
   %rval = icmp eq i32* %load, null
   ret i1 %rval
 }
;
   %rval = icmp eq i32 %load, 0
   ret i1 %rval
 }
 
 ; Make sure the above canonicalization does not trigger
 ; if the assume is control dependent on something else
 
 define i1 @nonnull3(i32** %a, i1 %control) {
-; SAME-LABEL: @nonnull3(
-; SAME-NEXT:  entry:
-; SAME-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
-; SAME-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
-; SAME-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
-; SAME:       taken:
-; SAME-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
-; SAME-NEXT:    ret i1 false
-; SAME:       not_taken:
-; SAME-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
-; SAME-NEXT:    ret i1 [[RVAL_2]]
+; FIXME: in the BUNDLES version we could duplicate the load and keep the assume nonnull.
+; DEFAULT-LABEL: @nonnull3(
+; DEFAULT-NEXT:  entry:
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; DEFAULT-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; DEFAULT-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; DEFAULT:       taken:
+; DEFAULT-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; DEFAULT-NEXT:    ret i1 false
+; DEFAULT:       not_taken:
+; DEFAULT-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
+; DEFAULT-NEXT:    ret i1 [[RVAL_2]]
+;
+; BUNDLES-LABEL: @nonnull3(
+; BUNDLES-NEXT:  entry:
+; BUNDLES-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; BUNDLES:       taken:
+; BUNDLES-NEXT:    ret i1 false
+; BUNDLES:       not_taken:
+; BUNDLES-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; BUNDLES-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
+; BUNDLES-NEXT:    ret i1 [[RVAL_2]]
 ;
 entry:
   %load = load i32*, i32** %a
   %cmp = icmp ne i32* %load, null
   br i1 %control, label %taken, label %not_taken
 taken:
   tail call void @llvm.assume(i1 %cmp)
   %rval = icmp eq i32* %load, null
   ret i1 %rval
 not_taken:
   %rval.2 = icmp sgt i32* %load, null
   ret i1 %rval.2
 }
 
 ; Make sure the above canonicalization does not trigger
 ; if the path from the load to the assume is potentially
 ; interrupted by an exception being thrown
 
 define i1 @nonnull4(i32** %a) {
-; SAME-LABEL: @nonnull4(
-; SAME-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
-; SAME-NEXT:    tail call void @escape(i32* [[LOAD]])
-; SAME-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
-; SAME-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
-; SAME-NEXT:    ret i1 false
+; DEFAULT-LABEL: @nonnull4(
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; DEFAULT-NEXT:    tail call void @escape(i32* [[LOAD]])
+; DEFAULT-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; DEFAULT-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; DEFAULT-NEXT:    ret i1 false
+;
+; BUNDLES-LABEL: @nonnull4(
+; BUNDLES-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; BUNDLES-NEXT:    tail call void @escape(i32* [[LOAD]])
+; BUNDLES-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]) ]
+; BUNDLES-NEXT:    ret i1 false
 ;
   %load = load i32*, i32** %a
   ;; This call may throw!
   tail call void @escape(i32* %load)
   %cmp = icmp ne i32* %load, null
   tail call void @llvm.assume(i1 %cmp)
   %rval = icmp eq i32* %load, null
   ret i1 %rval

 }
 
 ; PR37726 - https://bugs.llvm.org/show_bug.cgi?id=37726
 ; There's a loophole in eliminating a redundant assumption when
 ; we have conflicting assumptions. Verify that debuginfo doesn't
 ; get in the way of the fold.
 
 define void @debug_interference(i8 %x) {
-; SAME-LABEL: @debug_interference(
-; SAME-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[X:%.*]], 0
-; SAME-NEXT:    tail call void @llvm.assume(i1 false)
-; SAME-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META7:metadata !.*]], metadata !DIExpression()), [[DBG9:!dbg !.*]]
-; SAME-NEXT:    tail call void @llvm.assume(i1 false)
-; SAME-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META7]], metadata !DIExpression()), [[DBG9]]
-; SAME-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META7]], metadata !DIExpression()), [[DBG9]]
-; SAME-NEXT:    tail call void @llvm.assume(i1 [[CMP2]])
-; SAME-NEXT:    ret void
+; DEFAULT-LABEL: @debug_interference(
+; DEFAULT-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[X:%.*]], 0
+; DEFAULT-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META7:metadata !.*]], metadata !DIExpression()), [[DBG9:!dbg !.*]]
+; DEFAULT-NEXT:    tail call void @llvm.assume(i1 false)
+; DEFAULT-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META7]], metadata !DIExpression()), [[DBG9]]
+; DEFAULT-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META7]], metadata !DIExpression()), [[DBG9]]
+; DEFAULT-NEXT:    tail call void @llvm.assume(i1 [[CMP2]])
+; DEFAULT-NEXT:    ret void
+;
+; BUNDLES-LABEL: @debug_interference(
+; BUNDLES-NEXT:    [[CMP2:%.*]] = icmp ne i8 [[X:%.*]], 0
+; BUNDLES-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META6:metadata !.*]], metadata !DIExpression()), [[DBG8:!dbg !.*]]
+; BUNDLES-NEXT:    tail call void @llvm.assume(i1 false)
+; BUNDLES-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META6]], metadata !DIExpression()), [[DBG8]]
+; BUNDLES-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, [[META6]], metadata !DIExpression()), [[DBG8]]
+; BUNDLES-NEXT:    tail call void @llvm.assume(i1 [[CMP2]])
+; BUNDLES-NEXT:    ret void
 ;
   %cmp1 = icmp eq i8 %x, 0
   %cmp2 = icmp ne i8 %x, 0
   tail call void @llvm.assume(i1 %cmp1)
   tail call void @llvm.dbg.value(metadata i32 5, metadata !1, metadata !DIExpression()), !dbg !9
   tail call void @llvm.assume(i1 %cmp1)
   tail call void @llvm.dbg.value(metadata i32 5, metadata !1, metadata !DIExpression()), !dbg !9
   tail call void @llvm.assume(i1 %cmp2)
;
   %shuf = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
   %t2 = bitcast <4 x i8> %shuf to i32
   %t3 = icmp ult i32 %t2, 65536
   call void @llvm.assume(i1 %t3)
   ret i32 %t2
 }
 
 define i1 @nonnull3A(i32** %a, i1 %control) {
-; SAME-LABEL: @nonnull3A(
-; SAME-NEXT:  entry:
-; SAME-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
-; SAME-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
-; SAME:       taken:
-; SAME-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
-; SAME-NEXT:    call void @llvm.assume(i1 [[CMP]])
-; SAME-NEXT:    ret i1 true
-; SAME:       not_taken:
-; SAME-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
-; SAME-NEXT:    ret i1 [[RVAL_2]]
+; DEFAULT-LABEL: @nonnull3A(
+; DEFAULT-NEXT:  entry:
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; DEFAULT-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; DEFAULT:       taken:
+; DEFAULT-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; DEFAULT-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; DEFAULT-NEXT:    ret i1 true
+; DEFAULT:       not_taken:
+; DEFAULT-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
+; DEFAULT-NEXT:    ret i1 [[RVAL_2]]
+;
+; BUNDLES-LABEL: @nonnull3A(
+; BUNDLES-NEXT:  entry:
+; BUNDLES-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; BUNDLES:       taken:
+; BUNDLES-NEXT:    ret i1 true
+; BUNDLES:       not_taken:
+; BUNDLES-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; BUNDLES-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
+; BUNDLES-NEXT:    ret i1 [[RVAL_2]]
 ;
 entry:
   %load = load i32*, i32** %a
   %cmp = icmp ne i32* %load, null
   br i1 %control, label %taken, label %not_taken
 taken:
   call void @llvm.assume(i1 %cmp)
   ret i1 %cmp
 not_taken:
   call void @llvm.assume(i1 %cmp)
   %rval.2 = icmp sgt i32* %load, null
   ret i1 %rval.2
 }
 
 define i1 @nonnull3B(i32** %a, i1 %control) {
 ; SAME-LABEL: @nonnull3B(
 ; SAME-NEXT:  entry:
+; SAME-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
 ; SAME-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
 ; SAME:       taken:
-; SAME-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
-; SAME-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
-; SAME-NEXT:    call void @llvm.assume(i1 [[CMP]]) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 [[CMP]]) ]
+; SAME-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 true) ]
 ; SAME-NEXT:    ret i1 true
 ; SAME:       not_taken:
+; SAME-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[LOAD]]), "ignore"(i1 undef) ]
 ; SAME-NEXT:    ret i1 [[CONTROL]]
 ;
 entry:
   %load = load i32*, i32** %a
   %cmp = icmp ne i32* %load, null
   br i1 %control, label %taken, label %not_taken
 taken:
   call void @llvm.assume(i1 %cmp) ["nonnull"(i32* %load), "nonnull"(i1 %cmp)]
exit:
   %cmp4 = icmp eq i32 %x, 2
   tail call void @llvm.assume(i1 %cmp4)
   %cmp5 = icmp ugt i32 %y, 42
   tail call void @llvm.assume(i1 %cmp5)
   store i32 %x, i32* %p
   unreachable
 }
 
+define void @canonicalize_assume(i32* %0) {
+; SAME-LABEL: define {{[^@]+}}@canonicalize_assume
+; SAME-SAME: (i32* [[TMP0:%.*]])
+; SAME-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP0]], i64 8) ]
+; SAME-NEXT:    ret void
+;
+  %2 = getelementptr inbounds i32, i32* %0, i64 2
+  %3 = bitcast i32* %2 to i8*
+  call void @llvm.assume(i1 true) [ "align"(i8* %3, i64 16) ]
+  ret void
+}
+
 declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!5, !6, !7, !8}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, producer: "Me", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: null, retainedTypes: null, imports: null)
 !1 = !DILocalVariable(name: "", arg: 1, scope: !2, file: null, line: 1, type: null)
 !2 = distinct !DISubprogram(name: "debug", linkageName: "debug", scope: null, file: null, line: 0, type: null, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0)