[AssumeBundles] Preserve information in EarlyCSE

[Tyker]

Summary

this patch preserve information from various places in EarlyCSE into assume bundles.

[jdoerfert]

llvm/include/llvm/IR/KnowledgeRetention.h

38

Now that I see this I wonder why we have captial letters in the beginning.

llvm/test/Transforms/EarlyCSE/guards.ll

4

Can you please add the check lines and run the update script first in a local NFC commit. diff against that one and commit it just prior to this patch once accepted?

[Tyker]

Addressed comments.

[jdoerfert]

LGTM

llvm/include/llvm/IR/KnowledgeRetention.h

 class IntrinsicInst;
 
 /// Build a call to llvm.assume to preserve informations that can be derived
 /// from the given instruction.
 /// If no information derived from \p I, this call returns null.
 /// The returned instruction is not inserted anywhere.
 IntrinsicInst *buildAssumeFromInst(Instruction *I);
 
+/// Calls BuildAssumeFromInst and if the resulting llvm.assume is valid insert
+/// if before I. This is usually what need to be done to salvage the knowledge
+/// contained in the instruction I.
+void salvageKnowledge(Instruction* I);
+
 /// It is possible to have multiple Value for the argument of an attribute in
 /// the same llvm.assume on the same llvm::Value. This is rare but need to be
 /// dealt with.
 enum class AssumeQuery {
   Highest, ///< Take the highest value available.
   Lowest,  ///< Take the lowest value available.
 };
 

llvm/lib/IR/KnowledgeRetention.cpp

 IntrinsicInst *llvm::buildAssumeFromInst(Instruction *I) {
   if (!EnableKnowledgeRetention)
     return nullptr;
   AssumeBuilderState Builder(I->getModule());
   Builder.addInstruction(I);
   return Builder.build();
 }
 
+void llvm::salvageKnowledge(Instruction* I) {
+  if (Instruction* Intr = buildAssumeFromInst(I))
+    Intr->insertBefore(I);
+}
+
 static bool bundleHasArgument(const CallBase::BundleOpInfo &BOI,
                                unsigned Idx) {
   return BOI.End - BOI.Begin > Idx;
 }
 
 static Value *getValueFromBundleOpInfo(IntrinsicInst &Assume,
                                 const CallBase::BundleOpInfo &BOI,
                                 unsigned Idx) {

llvm/lib/Transforms/Scalar/EarlyCSE.cpp

 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/KnowledgeRetention.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
for (Instruction &Inst : make_early_inc_range(BB->getInstList())) {
     // Dead instructions should just be removed.
     if (isInstructionTriviallyDead(&Inst, &TLI)) {
       LLVM_DEBUG(dbgs() << "EarlyCSE DCE: " << Inst << '\n');
       if (!DebugCounter::shouldExecute(CSECounter)) {
         LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
         continue;
       }
 
+      salvageKnowledge(&Inst);
       salvageDebugInfoOrMarkUndef(Inst);
       removeMSSA(Inst);
       Inst.eraseFromParent();
       Changed = true;
       ++NumSimplify;
       continue;
     }
 
if (isGuard(&Inst)) {
         if (SimpleValue::canHandle(CondI)) {
           // Do we already know the actual value of this condition?
           if (auto *KnownCond = AvailableValues.lookup(CondI)) {
             // Is the condition known to be true?
             if (isa<ConstantInt>(KnownCond) &&
                 cast<ConstantInt>(KnownCond)->isOne()) {
               LLVM_DEBUG(dbgs()
                          << "EarlyCSE removing guard: " << Inst << '\n');
+              salvageKnowledge(&Inst);
               removeMSSA(Inst);
               Inst.eraseFromParent();
               Changed = true;
               continue;
             } else
               // Use the known value if it wasn't true.
               cast<CallInst>(Inst).setArgOperand(0, KnownCond);
           }
if (Value *V = SimplifyInstruction(&Inst, SQ)) {
         LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
       } else {
         bool Killed = false;
         if (!Inst.use_empty()) {
           Inst.replaceAllUsesWith(V);
           Changed = true;
         }
         if (isInstructionTriviallyDead(&Inst, &TLI)) {
+          salvageKnowledge(&Inst);
           removeMSSA(Inst);
           Inst.eraseFromParent();
           Changed = true;
           Killed = true;
         }
         if (Changed)
           ++NumSimplify;
         if (Killed)
if (SimpleValue::canHandle(&Inst)) {
                           << '\n');
         if (!DebugCounter::shouldExecute(CSECounter)) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
         if (auto *I = dyn_cast<Instruction>(V))
           I->andIRFlags(&Inst);
         Inst.replaceAllUsesWith(V);
+        salvageKnowledge(&Inst);
         removeMSSA(Inst);
         Inst.eraseFromParent();
         Changed = true;
         ++NumCSE;
         continue;
       }
 
       // Otherwise, just remember that this value is available.
if (MemInst.isValid() && MemInst.isLoad()) {
           LLVM_DEBUG(dbgs() << "EarlyCSE CSE LOAD: " << Inst
                             << "  to: " << *InVal.DefInst << '\n');
           if (!DebugCounter::shouldExecute(CSECounter)) {
             LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
             continue;
           }
           if (!Inst.use_empty())
             Inst.replaceAllUsesWith(Op);
+          salvageKnowledge(&Inst);
           removeMSSA(Inst);
           Inst.eraseFromParent();
           Changed = true;
           ++NumCSELoad;
           continue;
         }
       }
 
if (CallValue::canHandle(&Inst)) {
         LLVM_DEBUG(dbgs() << "EarlyCSE CSE CALL: " << Inst
                           << "  to: " << *InVal.first << '\n');
         if (!DebugCounter::shouldExecute(CSECounter)) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
         if (!Inst.use_empty())
           Inst.replaceAllUsesWith(InVal.first);
+        salvageKnowledge(&Inst);
         removeMSSA(Inst);
         Inst.eraseFromParent();
         Changed = true;
         ++NumCSECall;
         continue;
       }
 
       // Otherwise, remember that we have this instruction.
if (MemInst.isValid() && MemInst.isStore()) {
                     MemInst.getPointerOperand() ||
                 MSSA) &&
                "can't have an intervening store if not using MemorySSA!");
         LLVM_DEBUG(dbgs() << "EarlyCSE DSE (writeback): " << Inst << '\n');
         if (!DebugCounter::shouldExecute(CSECounter)) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
+        salvageKnowledge(&Inst);
         removeMSSA(Inst);
         Inst.eraseFromParent();
         Changed = true;
         ++NumDSE;
         // We can avoid incrementing the generation count since we were able
         // to eliminate this store.
         continue;
       }
if (Inst.mayWriteToMemory()) {
                  !LastStoreMemInst.isVolatile() &&
                  "Violated invariant");
           if (LastStoreMemInst.isMatchingMemLoc(MemInst)) {
             LLVM_DEBUG(dbgs() << "EarlyCSE DEAD STORE: " << *LastStore
                               << "  due to: " << Inst << '\n');
             if (!DebugCounter::shouldExecute(CSECounter)) {
               LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
             } else {
+              salvageKnowledge(&Inst);
               removeMSSA(*LastStore);
               LastStore->eraseFromParent();
               Changed = true;
               ++NumDSE;
               LastStore = nullptr;
             }
           }
           // fallthrough - we can exploit information about this store

llvm/test/Transforms/EarlyCSE/guards.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -early-cse < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 ; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 ; RUN: opt < %s -S -basicaa -early-cse-memssa --enable-knowledge-retention | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 
 declare void @llvm.experimental.guard(i1,...)
 
 declare void @llvm.assume(i1)
 
 define i32 @test0(i32* %ptr, i1 %cond) {
 ; We can do store to load forwarding over a guard, since it does not
 ; clobber memory
-; CHECK-LABEL: @test0(
-; CHECK-NEXT:    store i32 40, i32* [[PTR:%.*]]
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
-; CHECK-NEXT:    ret i32 40
+; NO_ASSUME-LABEL: @test0(
+; NO_ASSUME-NEXT:    store i32 40, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    ret i32 40
+;
+; USE_ASSUME-LABEL: @test0(
+; USE_ASSUME-NEXT:    store i32 40, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    ret i32 40
 ;
 
   store i32 40, i32* %ptr
   call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
   %rval = load i32, i32* %ptr
   ret i32 %rval
 }
 
 define i32 @test1(i32* %val, i1 %cond) {
 ; We can CSE loads over a guard, since it does not clobber memory
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[VAL0:%.*]] = load i32, i32* [[VAL:%.*]]
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: @test1(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[VAL:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: @test1(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[VAL:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[VAL]], i64 4), "nonnull"(i32* [[VAL]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
 
   %val0 = load i32, i32* %val
   call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
   %val1 = load i32, i32* %val
   %rval = sub i32 %val0, %val1
   ret i32 %rval
 }
;
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   ret void
 }
 
 define void @test08(i32 %a, i32 %b, i32* %ptr) {
 ; Check that we deal correctly with stores when removing guards in the same
 ; block in case when the condition is not recalculated.
-; CHECK-LABEL: @test08(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    store i32 100, i32* [[PTR:%.*]]
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
-; CHECK-NEXT:    store i32 400, i32* [[PTR]]
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test08(
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test08(
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    ret void
 ;
 
   %cmp = icmp eq i32 %a, %b
   store i32 100, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 200, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 300, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 400, i32* %ptr
   ret void
 }
 
 define void @test09(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Similar to test08, but with more control flow.
 ; TODO: Can we get rid of the store in the end of entry given that it is
 ; post-dominated by other stores?
-; CHECK-LABEL: @test09(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    store i32 100, i32* [[PTR:%.*]]
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
-; CHECK-NEXT:    store i32 400, i32* [[PTR]]
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    store i32 500, i32* [[PTR]]
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    store i32 600, i32* [[PTR]]
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test09(
+; NO_ASSUME-NEXT:  entry:
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; NO_ASSUME:       if.true:
+; NO_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; NO_ASSUME:       if.false:
+; NO_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test09(
+; USE_ASSUME-NEXT:  entry:
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; USE_ASSUME:       if.true:
+; USE_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; USE_ASSUME:       if.false:
+; USE_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    ret void
 ;
 
 entry:
   %cmp = icmp eq i32 %a, %b
   store i32 100, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 200, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
;
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   ret void
 }
 
 define void @test13(i32 %a, i32 %b, i32* %ptr) {
 ; Check that we deal correctly with stores when removing guards due to assume.
-; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    store i32 400, i32* [[PTR:%.*]]
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test13(
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test13(
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR:%.*]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    ret void
 ;
 
   %cmp = icmp eq i32 %a, %b
   call void @llvm.assume(i1 %cmp)
   store i32 100, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 200, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 300, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 400, i32* %ptr
   ret void
 }
 
 define void @test14(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Similar to test13, but with more control flow.
 ; TODO: Can we get rid of the store in the end of entry given that it is
 ; post-dominated by other stores?
-; CHECK-LABEL: @test14(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    store i32 400, i32* [[PTR:%.*]]
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    store i32 500, i32* [[PTR]]
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    store i32 600, i32* [[PTR]]
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test14(
+; NO_ASSUME-NEXT:  entry:
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; NO_ASSUME:       if.true:
+; NO_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; NO_ASSUME:       if.false:
+; NO_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test14(
+; USE_ASSUME-NEXT:  entry:
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR:%.*]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; USE_ASSUME:       if.true:
+; USE_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; USE_ASSUME:       if.false:
+; USE_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    ret void
 ;
 
 entry:
   %cmp = icmp eq i32 %a, %b
   call void @llvm.assume(i1 %cmp)
   store i32 100, i32* %ptr
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
   store i32 200, i32* %ptr

llvm/test/Transforms/EarlyCSE/invariant-loads.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -early-cse < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 ; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 ; RUN: opt -S -basicaa -early-cse-memssa --enable-knowledge-retention < %s | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 
 declare void @clobber_and_use(i32)
 
 define void @f_0(i32* %ptr) {
-; CHECK-LABEL: @f_0(
-; CHECK-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @f_0(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_0(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
 ;
 
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
   %val1 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val1)
   %val2 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val2)
   ret void
 }
 
 define void @f_1(i32* %ptr) {
 ; We can forward invariant loads to non-invariant loads.
-; CHECK-LABEL: @f_1(
-; CHECK-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @f_1(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_1(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
 ;
 
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
   %val1 = load i32, i32* %ptr
   call void @clobber_and_use(i32 %val1)
   ret void
 }
 
 define void @f_2(i32* %ptr) {
 ; We can forward a non-invariant load into an invariant load.
-; CHECK-LABEL: @f_2(
-; CHECK-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]]
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @f_2(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_2(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
 ;
 
   %val0 = load i32, i32* %ptr
   call void @clobber_and_use(i32 %val0)
   %val1 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val1)
   ret void
 }
 
 define void @f_3(i1 %cond, i32* %ptr) {
-; CHECK-LABEL: @f_3(
-; CHECK-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
-; CHECK:       left:
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
-; CHECK-NEXT:    ret void
-; CHECK:       right:
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @f_3(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; NO_ASSUME:       left:
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+; NO_ASSUME:       right:
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_3(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; USE_ASSUME:       left:
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
+; USE_ASSUME:       right:
+; USE_ASSUME-NEXT:    ret void
 ;
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
   br i1 %cond, label %left, label %right
 
 
 left:
   %val1 = load i32, i32* %ptr
merge:
   ret void
 }
 
 ; By assumption, the call can't change contents of p
 ; LangRef is a bit unclear about whether the store is reachable, so
 ; for the moment we chose to be conservative and just assume it's valid
 ; to restore the same unchanging value.
 define void @test_dse1(i32* %p) {
-; CHECK-LABEL: @test_dse1(
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[V1]])
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test_dse1(
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test_dse1(
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret void
 ;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber_and_use(i32 %v1)
   store i32 %v1, i32* %p
   ret void
 }
 
 ; By assumption, v1 must equal v2 (TODO)
 define void @test_false_negative_dse2(i32* %p, i32 %v2) {
 ; CHECK-LABEL: @test_false_negative_dse2(
 ; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
 ; CHECK-NEXT:    call void @clobber_and_use(i32 [[V1]])
 ; CHECK-NEXT:    store i32 [[V2:%.*]], i32* [[P]]
 ; CHECK-NEXT:    ret void
 ;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber_and_use(i32 %v1)
   store i32 %v2, i32* %p
   ret void
 }
 
 ; If we remove the load, we still start an invariant scope since
 ; it lets us remove later loads not explicitly marked invariant
 define void @test_scope_start_without_load(i32* %p) {
-; CHECK-LABEL: @test_scope_start_without_load(
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[ADD]])
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[V1]])
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test_scope_start_without_load(
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]]
+; NO_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test_scope_start_without_load(
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    ret void
 ;
   %v1 = load i32, i32* %p
   %v2 = load i32, i32* %p, !invariant.load !{}
   %add = add i32 %v1, %v2
   call void @clobber_and_use(i32 %add)
   %v3 = load i32, i32* %p
   call void @clobber_and_use(i32 %v3)
   ret void
 }
 
 ; If we already have an invariant scope, don't want to start a new one
 ; with a potentially greater generation.  This hides the earlier invariant
 ; load
 define void @test_scope_restart(i32* %p) {
-; CHECK-LABEL: @test_scope_restart(
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[V1]])
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[ADD]])
-; CHECK-NEXT:    call void @clobber_and_use(i32 [[V1]])
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test_scope_restart(
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test_scope_restart(
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    ret void
 ;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber_and_use(i32 %v1)
   %v2 = load i32, i32* %p, !invariant.load !{}
   %add = add i32 %v1, %v2
   call void @clobber_and_use(i32 %add)
   %v3 = load i32, i32* %p
   call void @clobber_and_use(i32 %v3)
   ret void
 }

llvm/test/Transforms/EarlyCSE/invariant.start.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; RUN: opt < %s -S -early-cse | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 ; RUN: opt < %s -S -early-cse --enable-knowledge-retention | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 ; RUN: opt < %s -S -passes=early-cse | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 
 declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
 declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
 
 ; Check that we do load-load forwarding over invariant.start, since it does not
 ; clobber memory
 define i8 @test_bypass1(i8 *%P) {
-; CHECK-LABEL: define {{[^@]+}}@test_bypass1
-; CHECK-SAME: (i8* [[P:%.*]])
-; CHECK-NEXT:    [[V1:%.*]] = load i8, i8* [[P]]
-; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
-; CHECK-NEXT:    ret i8 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_bypass1
+; NO_ASSUME-SAME: (i8* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i8, i8* [[P]]
+; NO_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; NO_ASSUME-NEXT:    ret i8 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_bypass1
+; USE_ASSUME-SAME: (i8* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i8, i8* [[P]]
+; USE_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[P]], i64 1), "nonnull"(i8* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i8 0
 ;
 
   %V1 = load i8, i8* %P
   %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
   %V2 = load i8, i8* %P
   %Diff = sub i8 %V1, %V2
   ret i8 %Diff
 }
 
 
 ; Trivial Store->load forwarding over invariant.start
 define i8 @test_bypass2(i8 *%P) {
-; CHECK-LABEL: define {{[^@]+}}@test_bypass2
-; CHECK-SAME: (i8* [[P:%.*]])
-; CHECK-NEXT:    store i8 42, i8* [[P]]
-; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
-; CHECK-NEXT:    ret i8 42
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_bypass2
+; NO_ASSUME-SAME: (i8* [[P:%.*]])
+; NO_ASSUME-NEXT:    store i8 42, i8* [[P]]
+; NO_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; NO_ASSUME-NEXT:    ret i8 42
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_bypass2
+; USE_ASSUME-SAME: (i8* [[P:%.*]])
+; USE_ASSUME-NEXT:    store i8 42, i8* [[P]]
+; USE_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[P]], i64 1), "nonnull"(i8* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i8 42
 ;
 
   store i8 42, i8* %P
   %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
   %V1 = load i8, i8* %P
   ret i8 %V1
 }
 
 ; We can DSE over invariant.start calls, since the first store to
 ; %P is valid, and the second store is actually unreachable based on semantics
 ; of invariant.start.
 define void @test_bypass3(i8* %P) {
-; CHECK-LABEL: define {{[^@]+}}@test_bypass3
-; CHECK-SAME: (i8* [[P:%.*]])
-; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
-; CHECK-NEXT:    store i8 60, i8* [[P]]
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_bypass3
+; NO_ASSUME-SAME: (i8* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; NO_ASSUME-NEXT:    store i8 60, i8* [[P]]
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_bypass3
+; USE_ASSUME-SAME: (i8* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[P]], i64 1), "nonnull"(i8* [[P]]) ]
+; USE_ASSUME-NEXT:    store i8 60, i8* [[P]]
+; USE_ASSUME-NEXT:    ret void
 ;
 
   store i8 50, i8* %P
   %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
   store i8 60, i8* %P
   ret void
 }
 

 }
 
 
 declare void @clobber()
 declare {}* @llvm.invariant.start.p0i32(i64 %size, i32* nocapture %ptr)
 declare void @llvm.invariant.end.p0i32({}*, i64, i32* nocapture) nounwind
 
 define i32 @test_before_load(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_before_load
-; CHECK-SAME: (i32* [[P:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_before_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_before_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 define i32 @test_before_clobber(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_before_clobber
-; CHECK-SAME: (i32* [[P:%.*]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_before_clobber
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_before_clobber
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 define i32 @test_duplicate_scope(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_duplicate_scope
-; CHECK-SAME: (i32* [[P:%.*]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    [[TMP2:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_duplicate_scope
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    [[TMP2:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_duplicate_scope
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    [[TMP2:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 define i32 @test_unanalzyable_load(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_unanalzyable_load
-; CHECK-SAME: (i32* [[P:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_unanalzyable_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_unanalzyable_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   %v1 = load i32, i32* %p
   call void @clobber()
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
;
   call void @clobber()
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 define i32 @test_merge(i32* %p, i1 %cnd) {
-; CHECK-LABEL: define {{[^@]+}}@test_merge
-; CHECK-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
-; CHECK:       taken:
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_merge
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; NO_ASSUME:       taken:
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_merge
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; USE_ASSUME:       taken:
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   br i1 %cnd, label %merge, label %taken
 
 taken:
   call void @clobber()
   br label %merge
taken:
   br label %merge
 merge:
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 define i32 @test_merge_unanalyzable_load(i32* %p, i1 %cnd) {
-; CHECK-LABEL: define {{[^@]+}}@test_merge_unanalyzable_load
-; CHECK-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
-; CHECK:       taken:
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_merge_unanalyzable_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; NO_ASSUME:       taken:
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_merge_unanalyzable_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; USE_ASSUME:       taken:
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   %v1 = load i32, i32* %p
   br i1 %cnd, label %merge, label %taken
 
 taken:
   call void @clobber()
   br label %merge
 merge:
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 define void @test_dse_before_load(i32* %p, i1 %cnd) {
-; CHECK-LABEL: define {{[^@]+}}@test_dse_before_load
-; CHECK-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_dse_before_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_dse_before_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret void
 ;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
   store i32 %v1, i32* %p
   ret void
 }
 
 define void @test_dse_after_load(i32* %p, i1 %cnd) {
-; CHECK-LABEL: define {{[^@]+}}@test_dse_after_load
-; CHECK-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_dse_after_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_dse_after_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret void
 ;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   store i32 %v1, i32* %p
   ret void
 }
 
;
   %v2 = load i32, i32* %p
   call void @llvm.invariant.end.p0i32({}* %scope, i64 4, i32* %p)
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }
 
 ; Invariant load defact starts an invariant.start scope of the appropriate size
 define i32 @test_invariant_load_scope(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_invariant_load_scope
-; CHECK-SAME: (i32* [[P:%.*]])
-; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]], !invariant.load !0
-; CHECK-NEXT:    call void @clobber()
-; CHECK-NEXT:    ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_invariant_load_scope
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_invariant_load_scope
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
 ;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber()
   %v2 = load i32, i32* %p
   %sub = sub i32 %v1, %v2
   ret i32 %sub
 }