Index: lib/Analysis/CodeMetrics.cpp =================================================================== --- lib/Analysis/CodeMetrics.cpp +++ lib/Analysis/CodeMetrics.cpp @@ -76,20 +76,12 @@ SmallPtrSet Visited; SmallVector Worklist; - for (auto &AssumeVH : AC->assumptions()) { - if (!AssumeVH) - continue; - Instruction *I = cast(AssumeVH); - - // Filter out call sites outside of the loop so we don't do a function's - // worth of work for each of its loops (and, in the common case, ephemeral - // values in the loop are likely due to @llvm.assume calls in the loop). - if (!L->contains(I->getParent())) - continue; - - if (EphValues.insert(I).second) - appendSpeculatableOperands(I, Visited, Worklist); - } + for (auto &B : L->blocks()) + for (auto &I : *B) + if (auto *II = dyn_cast(&I)) + if (II->getIntrinsicID() == Intrinsic::assume && + EphValues.insert(II).second) + appendSpeculatableOperands(II, Visited, Worklist); completeEphemeralValues(Visited, Worklist, EphValues); } @@ -100,16 +92,12 @@ SmallPtrSet Visited; SmallVector Worklist; - for (auto &AssumeVH : AC->assumptions()) { - if (!AssumeVH) - continue; - Instruction *I = cast(AssumeVH); - assert(I->getParent()->getParent() == F && - "Found assumption for the wrong function!"); - - if (EphValues.insert(I).second) - appendSpeculatableOperands(I, Visited, Worklist); - } + for (auto &B : *F) + for (auto &I : B) + if (auto *II = dyn_cast(&I)) + if (II->getIntrinsicID() == Intrinsic::assume && + EphValues.insert(II).second) + appendSpeculatableOperands(II, Visited, Worklist); completeEphemeralValues(Visited, Worklist, EphValues); } Index: lib/Analysis/LazyValueInfo.cpp =================================================================== --- lib/Analysis/LazyValueInfo.cpp +++ lib/Analysis/LazyValueInfo.cpp @@ -962,10 +962,12 @@ if (!BBI) return; - for (auto &AssumeVH : AC->assumptions()) { - if (!AssumeVH) + for (auto *U : Val->users()) { + const IntrinsicInst *I = dyn_cast(U); + if (!I) + continue; + if (I->getIntrinsicID() != Intrinsic::assume) continue; - auto *I = cast(AssumeVH); if (!isValidAssumeForContext(I, BBI, DT)) continue; Index: lib/Analysis/ValueTracking.cpp =================================================================== --- lib/Analysis/ValueTracking.cpp +++ lib/Analysis/ValueTracking.cpp @@ -527,22 +527,15 @@ unsigned BitWidth = KnownZero.getBitWidth(); - for (auto &AssumeVH : Q.AC->assumptions()) { - if (!AssumeVH) + for (auto *U : V->users()) { + const IntrinsicInst *I = dyn_cast(U); + if (!I) + continue; + if (I->getIntrinsicID() != Intrinsic::assume) continue; - CallInst *I = cast(AssumeVH); - assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() && - "Got assumption for the wrong function!"); if (Q.isExcluded(I)) continue; - // Warning: This loop can end up being somewhat performance sensetive. - // We're running this loop for once for each value queried resulting in a - // runtime of ~O(#assumes * #values). - - assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && - "must be an assume intrinsic"); - Value *Arg = I->getArgOperand(0); if (Arg == V && isValidAssumeForContext(I, Q.CxtI, Q.DT)) { @@ -552,6 +545,10 @@ return; } + // Note that the patterns below need to be kept in sync with the code + // in InstCombiner::visitCallInst that adds relevant values to each + // assume's operand bundles. + // The remaining tests are all recursive, so bail out if we hit the limit. if (Depth == MaxDepth) continue; Index: lib/Transforms/InstCombine/InstCombineCalls.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineCalls.cpp +++ lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2501,6 +2501,78 @@ if (KnownOne.isAllOnesValue()) return eraseInstFromFunction(*II); + // For assumptions, add to the associated operand bundle the values to which + // the assumption might apply. + // Note: This code must be kept in-sync with the code in + // computeKnownBitsFromAssume in ValueTracking. + SmallVector Affected; + auto AddAffected = [&Affected](Value *V) { + if (isa(V)) { + Affected.push_back(V); + } else if (auto *I = dyn_cast(V)) { + Affected.push_back(I); + + if (I->getOpcode() == Instruction::BitCast || + I->getOpcode() == Instruction::PtrToInt) { + V = I->getOperand(0); + if (isa(V) || isa(V)) + Affected.push_back(V); + } + } + }; + + CmpInst::Predicate Pred; + if (match(IIOperand, m_ICmp(Pred, m_Value(A), m_Value(B)))) { + AddAffected(A); + AddAffected(B); + + if (Pred == ICmpInst::ICMP_EQ) { + // For equality comparisons, we handle the case of bit inversion. + auto AddAffectedFromEq = [&AddAffected](Value *V) { + Value *A; + if (match(V, m_Not(m_Value(A)))) { + AddAffected(A); + V = A; + } + + Value *B; + ConstantInt *C; + if (match(V, + m_CombineOr(m_And(m_Value(A), m_Value(B)), + m_CombineOr(m_Or(m_Value(A), m_Value(B)), + m_Xor(m_Value(A), m_Value(B)))))) { + AddAffected(A); + AddAffected(B); + } else if (match(V, + m_CombineOr(m_Shl(m_Value(A), m_ConstantInt(C)), + m_CombineOr(m_LShr(m_Value(A), m_ConstantInt(C)), + m_AShr(m_Value(A), + m_ConstantInt(C)))))) { + AddAffected(A); + } + }; + + AddAffectedFromEq(A); + AddAffectedFromEq(B); + } + } + + // If the list of affected values is the same as the existing list then + // there's nothing more to do here. + if (!Affected.empty()) + if (auto OB = CI.getOperandBundle("affected")) + if (Affected.size() == OB.getValue().Inputs.size() && + std::equal(Affected.begin(), Affected.end(), + OB.getValue().Inputs.begin())) + Affected.clear(); + + if (!Affected.empty()) { + Builder->CreateCall(AssumeIntrinsic, IIOperand, + OperandBundleDef("affected", Affected), + II->getName()); + return eraseInstFromFunction(*II); + } + break; } case Intrinsic::experimental_gc_relocate: { Index: lib/Transforms/Scalar/AlignmentFromAssumptions.cpp =================================================================== --- lib/Transforms/Scalar/AlignmentFromAssumptions.cpp +++ lib/Transforms/Scalar/AlignmentFromAssumptions.cpp @@ -425,9 +425,12 @@ NewSrcAlignments.clear(); bool Changed = false; - for (auto &AssumeVH : AC.assumptions()) - if (AssumeVH) - Changed |= processAssumption(cast(AssumeVH)); + + for (auto &B : F) + for (auto &I : B) + if (auto *II = dyn_cast(&I)) + if (II->getIntrinsicID() == Intrinsic::assume) + Changed |= processAssumption(II); return Changed; } Index: test/Transforms/CorrelatedValuePropagation/conflict.ll =================================================================== --- test/Transforms/CorrelatedValuePropagation/conflict.ll +++ test/Transforms/CorrelatedValuePropagation/conflict.ll @@ -26,7 +26,7 @@ define i8 @test2(i8 %a) { ; CHECK-LABEL: @test2 %cmp1 = icmp eq i8 %a, 5 - call void @llvm.assume(i1 %cmp1) + call void @llvm.assume(i1 %cmp1) [ "affected"(i8 %a) ] %cmp2 = icmp eq i8 %a, 3 ; CHECK: br i1 false, label %dead, label %exit br i1 %cmp2, label %dead, label %exit @@ -43,7 +43,7 @@ dead: %cmp2 = icmp eq i8 %a, 3 ; CHECK: call void @llvm.assume(i1 false) - call void @llvm.assume(i1 %cmp2) + call void @llvm.assume(i1 %cmp2) [ "affected"(i8 %a) ] ret i8 %a exit: ret i8 0 Index: test/Transforms/InstCombine/assume-redundant.ll =================================================================== --- test/Transforms/InstCombine/assume-redundant.ll +++ test/Transforms/InstCombine/assume-redundant.ll @@ -11,7 +11,7 @@ define void @_Z3fooR1s(%struct.s* nocapture readonly dereferenceable(8) %x) #0 { ; CHECK-LABEL: @_Z3fooR1s -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %maskcond) [ "affected"(i64 %maskedptr, i64 %ptrint, double* %{{.*}}) ] ; CHECK-NOT: call void @llvm.assume entry: Index: test/Transforms/InstCombine/assume.ll =================================================================== --- test/Transforms/InstCombine/assume.ll +++ test/Transforms/InstCombine/assume.ll @@ -11,7 +11,7 @@ ; been removed: ; CHECK-LABEL: @foo1 ; CHECK-DAG: load i32, i32* %a, align 32 -; CHECK-DAG: call void @llvm.assume +; CHECK-DAG: call void @llvm.assume(i1 %maskcond) [ "affected"(i64 %maskedptr, i64 %ptrint, i32* %a) ] ; CHECK: ret i32 %ptrint = ptrtoint i32* %a to i64 @@ -28,7 +28,7 @@ ; Same check as in @foo1, but make sure it works if the assume is first too. ; CHECK-LABEL: @foo2 ; CHECK-DAG: load i32, i32* %a, align 32 -; CHECK-DAG: call void @llvm.assume +; CHECK-DAG: call void @llvm.assume(i1 %maskcond) [ "affected"(i64 %maskedptr, i64 %ptrint, i32* %a) ] ; CHECK: ret i32 %ptrint = ptrtoint i32* %a to i64 @@ -51,7 +51,7 @@ ; CHECK: ret i32 4 %cmp = icmp eq i32 %a, 4 - tail call void @llvm.assume(i1 %cmp) + tail call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ret i32 %a } @@ -93,7 +93,7 @@ %and1 = and i32 %a, 3 ; CHECK-LABEL: @bar1 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ] ; CHECK: ret i32 1 %and = and i32 %a, 7 @@ -107,7 +107,7 @@ define i32 @bar2(i32 %a) #0 { entry: ; CHECK-LABEL: @bar2 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ] ; CHECK: ret i32 1 %and = and i32 %a, 7 @@ -125,7 +125,7 @@ ; Don't be fooled by other assumes around. ; CHECK-LABEL: @bar3 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ] ; CHECK: ret i32 1 tail call void @llvm.assume(i1 %x) @@ -145,8 +145,8 @@ %and1 = and i32 %b, 3 ; CHECK-LABEL: @bar4 -; CHECK: call void @llvm.assume -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ] +; CHECK: call void @llvm.assume(i1 %cmp2) [ "affected"(i32 %a, i32 %b) ] ; CHECK: ret i32 1 %and = and i32 %a, 7 @@ -167,7 +167,7 @@ ret i32 %conv ; CHECK-LABEL: @icmp1 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ; CHECK: ret i32 1 } @@ -182,7 +182,7 @@ ret i32 %lnot.ext ; CHECK-LABEL: @icmp2 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ; CHECK: ret i32 0 } @@ -217,7 +217,7 @@ ; CHECK-LABEL: @nonnull2 ; CHECK-NOT: !nonnull -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %load) ] } ; Make sure the above canonicalization does not trigger @@ -236,7 +236,7 @@ ; CHECK-LABEL: @nonnull3 ; CHECK-NOT: !nonnull -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32* %load) ] } ; Make sure the above canonicalization does not trigger @@ -254,7 +254,7 @@ ; CHECK-LABEL: @nonnull4 ; CHECK-NOT: !nonnull -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32* %load) ] } define i1 @nonnull5(i32** %a) { Index: test/Transforms/InstCombine/assume2.ll =================================================================== --- test/Transforms/InstCombine/assume2.ll +++ test/Transforms/InstCombine/assume2.ll @@ -9,7 +9,7 @@ define i32 @test1(i32 %a) #0 { entry: ; CHECK-LABEL: @test1 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ] ; CHECK: ret i32 5 %and = and i32 %a, 15 @@ -24,7 +24,7 @@ define i32 @test2(i32 %a) #0 { entry: ; CHECK-LABEL: @test2 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a.not, i32 %a) ] ; CHECK: ret i32 2 %and = and i32 %a, 15 @@ -40,7 +40,7 @@ define i32 @test3(i32 %a) #0 { entry: ; CHECK-LABEL: @test3 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v, i32 %a) ] ; CHECK: ret i32 5 %v = or i32 %a, 4294967280 @@ -55,7 +55,7 @@ define i32 @test4(i32 %a) #0 { entry: ; CHECK-LABEL: @test4 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a.not, i32 %a) ] ; CHECK: ret i32 2 %v = or i32 %a, 4294967280 @@ -71,7 +71,7 @@ define i32 @test5(i32 %a) #0 { entry: ; CHECK-LABEL: @test5 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ; CHECK: ret i32 4 %v = xor i32 %a, 1 @@ -86,7 +86,7 @@ define i32 @test6(i32 %a) #0 { entry: ; CHECK-LABEL: @test6 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v.mask, i32 %a) ] ; CHECK: ret i32 5 %v = shl i32 %a, 2 @@ -101,7 +101,7 @@ define i32 @test7(i32 %a) #0 { entry: ; CHECK-LABEL: @test7 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v.mask, i32 %a) ] ; CHECK: ret i32 20 %v = lshr i32 %a, 2 @@ -116,7 +116,7 @@ define i32 @test8(i32 %a) #0 { entry: ; CHECK-LABEL: @test8 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v.mask, i32 %a) ] ; CHECK: ret i32 20 %v = lshr i32 %a, 2 @@ -131,7 +131,7 @@ define i32 @test9(i32 %a) #0 { entry: ; CHECK-LABEL: @test9 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ; CHECK: ret i32 0 %cmp = icmp sgt i32 %a, 5 @@ -145,7 +145,7 @@ define i32 @test10(i32 %a) #0 { entry: ; CHECK-LABEL: @test10 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ; CHECK: ret i32 -2147483648 %cmp = icmp sle i32 %a, -2 @@ -159,7 +159,7 @@ define i32 @test11(i32 %a) #0 { entry: ; CHECK-LABEL: @test11 -; CHECK: call void @llvm.assume +; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] ; CHECK: ret i32 0 %cmp = icmp ule i32 %a, 256 Index: test/Transforms/InstSimplify/add-mask.ll =================================================================== --- test/Transforms/InstSimplify/add-mask.ll +++ test/Transforms/InstSimplify/add-mask.ll @@ -46,7 +46,7 @@ %b = load i32, i32* @B %b.and = and i32 %b, 1 %b.cnd = icmp eq i32 %b.and, 1 - call void @llvm.assume(i1 %b.cnd) + call void @llvm.assume(i1 %b.cnd) [ "affected"(i32 %b.and, i32 %b) ] %rhs = add i32 %a, %b %and = and i32 %a, %rhs Index: test/Transforms/JumpThreading/assume-edge-dom.ll =================================================================== --- test/Transforms/JumpThreading/assume-edge-dom.ll +++ test/Transforms/JumpThreading/assume-edge-dom.ll @@ -14,12 +14,12 @@ taken: %res1 = call i8* @escape() %a = icmp eq i8* %res1, null - tail call void @llvm.assume(i1 %a) + tail call void @llvm.assume(i1 %a) [ "affected"(i8* %res1) ] br label %done not_taken: %res2 = call i8* @escape() %b = icmp ne i8* %res2, null - tail call void @llvm.assume(i1 %b) + tail call void @llvm.assume(i1 %b) [ "affected"(i8* %res2) ] br label %done ; An assume that can be used to simplify this comparison dominates each Index: test/Transforms/JumpThreading/assume.ll =================================================================== --- test/Transforms/JumpThreading/assume.ll +++ test/Transforms/JumpThreading/assume.ll @@ -6,7 +6,7 @@ define i32 @test1(i32 %a, i32 %b) #0 { entry: %cmp = icmp sgt i32 %a, 5 - tail call void @llvm.assume(i1 %cmp) + tail call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] %cmp1 = icmp sgt i32 %b, 1234 br i1 %cmp1, label %if.then, label %if.else @@ -36,7 +36,7 @@ define i32 @test2(i32 %a) #0 { entry: %cmp = icmp sgt i32 %a, 5 - tail call void @llvm.assume(i1 %cmp) + tail call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ] %cmp1 = icmp sgt i32 %a, 3 br i1 %cmp1, label %if.then, label %return Index: test/Transforms/NaryReassociate/NVPTX/nary-gep.ll =================================================================== --- test/Transforms/NaryReassociate/NVPTX/nary-gep.ll +++ test/Transforms/NaryReassociate/NVPTX/nary-gep.ll @@ -75,10 +75,10 @@ ; CHECK-LABEL: @reassociate_gep_assume( ; assume(j >= 0) %cmp = icmp sgt i32 %j, -1 - call void @llvm.assume(i1 %cmp) + call void @llvm.assume(i1 %cmp) [ "affected"(i32 %j) ] %1 = add i32 %i, %j %cmp2 = icmp sgt i32 %1, -1 - call void @llvm.assume(i1 %cmp2) + call void @llvm.assume(i1 %cmp2) [ "affected"(i32 %1) ] %idxprom.j = zext i32 %j to i64 %2 = getelementptr float, float* %a, i64 %idxprom.j Index: test/Transforms/SimplifyCFG/switch-dead-default.ll =================================================================== --- test/Transforms/SimplifyCFG/switch-dead-default.ll +++ test/Transforms/SimplifyCFG/switch-dead-default.ll @@ -91,7 +91,7 @@ ; CHECK-LABEL: @test5 ; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false %cmp = icmp ult i8 %a, 2 - call void @llvm.assume(i1 %cmp) + call void @llvm.assume(i1 %cmp) [ "affected"(i8 %a) ] switch i8 %a, label %default [i8 1, label %true i8 0, label %false] true: @@ -112,7 +112,7 @@ ; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false %and = and i8 %a, 254 %cmp = icmp eq i8 %and, 254 - call void @llvm.assume(i1 %cmp) + call void @llvm.assume(i1 %cmp) [ "affected"(i8 %and, i8 %a) ] switch i8 %a, label %default [i8 255, label %true i8 254, label %false] true: @@ -134,7 +134,7 @@ ; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false %and = and i8 %a, 254 %cmp = icmp eq i8 %and, 254 - call void @llvm.assume(i1 %cmp) + call void @llvm.assume(i1 %cmp) [ "affected"(i8 %and, i8 %a) ] switch i8 %a, label %default [i8 255, label %true i8 254, label %false i8 0, label %also_dead] @@ -162,7 +162,7 @@ ; CHECK: switch i8 %and = and i8 %a, 254 %cmp = icmp eq i8 %and, undef - call void @llvm.assume(i1 %cmp) + call void @llvm.assume(i1 %cmp) [ "affected"(i8 %and, i8 %a) ] switch i8 %a, label %default [i8 255, label %true i8 254, label %false] true: