[ScalarEvolution] Add an additional bailout to avoid NOT of pointer.

[efriedma]

Summary

It's possible in some cases for the LHS to be a pointer where the RHS is not. This isn't directly possible for an icmp, but the analysis mixes up operands of different icmp expressions in some cases.

I haven't managed to reduce a testcase I can commit; the testcase in the bug depends on sanitizer passes, and the obvious reduction methods didn't work for me.

Also add an assertion to getNotSCEV() to make tracking down this sort of issue a bit easier in the future.

Fixes https://bugs.llvm.org/show_bug.cgi?id=51787 .

[efriedma]

@reames will post a revised patch.

llvm/lib/Analysis/ScalarEvolution.cpp

if (!AddRHS || AddRHS->getNumOperands() != 2 ||
       !AddRHS->getOperand(0)->isAllOnesValue())
     return nullptr;
 
   return AddRHS->getOperand(1);
 }
 
 /// Return a SCEV corresponding to ~V = -1-V
 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
+  assert(!V->getType()->isPointerTy() && "Can't negate pointer");
+
   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
     return getConstant(
                 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
 
   // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
   if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(V)) {
     auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
       SmallVector<const SCEV *, 2> MatchedOperands;
if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
     // do this if it would break canonical constant/addrec ordering.
     if (!isa<SCEVConstant>(RHS) && !isa<SCEVAddRecExpr>(LHS))
       return isImpliedCondOperands(FoundPred, RHS, LHS, FoundLHS, FoundRHS,
                                    Context);
     if (!isa<SCEVConstant>(FoundRHS) && !isa<SCEVAddRecExpr>(FoundLHS))
       return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS, Context);
 
     // Don't try to getNotSCEV pointers.
-    if (LHS->getType()->isPointerTy() || FoundLHS->getType()->isPointerTy())
+    if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy() ||
+        FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
       return false;
 
     // There's no clear preference between forms 3. and 4., try both.
     return isImpliedCondOperands(FoundPred, getNotSCEV(LHS), getNotSCEV(RHS),
                                  FoundLHS, FoundRHS, Context) ||
            isImpliedCondOperands(Pred, LHS, RHS, getNotSCEV(FoundLHS),
                                  getNotSCEV(FoundRHS), Context);
   }