[ValueTracking] Fix endless recursion in isKnownNonZero()

[Ka-Ka]

Summary

The isKnownNonZero() function have checks that abort the recursion when
it reach the specified max depth. However one of the recursive calls was
placed before the max depth check was done, resulting in a endless
recursion that eventually triggered a segmentation fault.

Fixed the problem by moving the max depth check above the first
recursive call.

[bjope]

test/Transforms/CorrelatedValuePropagation/pointer.ll

2

Remove the -mtriple if possible.
If the -mtriple is needed, then the test should to go into a target specific subdir with a lit.local.cfg.

[Ka-Ka]

Updated testcase according to comment.

[hfinkel]

LGTM

lib/Analysis/ValueTracking.cpp

if (MDNode *Ranges = I->getMetadata(LLVMContext::MD_range)) {
       if (auto *Ty = dyn_cast<IntegerType>(V->getType())) {
         const APInt ZeroValue(Ty->getBitWidth(), 0);
         if (rangeMetadataExcludesValue(Ranges, ZeroValue))
           return true;
       }
     }
   }
 
+  // Some of the tests below are recursive, so bail out if we hit the limit.
+  if (Depth++ >= MaxDepth)
+    return false;
+
   // Check for pointer simplifications.
   if (V->getType()->isPointerTy()) {
     // Alloca never returns null, malloc might.
     if (isa<AllocaInst>(V) && Q.DL.getAllocaAddrSpace() == 0)
       return true;
 
     // A byval, inalloca, or nonnull argument is never null.
     if (const Argument *A = dyn_cast<Argument>(V))
       if (A->hasByValOrInAllocaAttr() || A->hasNonNullAttr())
         return true;
 
     // A Load tagged with nonnull metadata is never null.
     if (const LoadInst *LI = dyn_cast<LoadInst>(V))
       if (LI->getMetadata(LLVMContext::MD_nonnull))
         return true;
 
     if (auto CS = ImmutableCallSite(V)) {
       if (CS.isReturnNonNull())
         return true;
       if (const auto *RP = getArgumentAliasingToReturnedPointer(CS))
-        return isKnownNonZero(RP, Depth + 1, Q);
+        return isKnownNonZero(RP, Depth, Q);
     }
   }
 
-  // The remaining tests are all recursive, so bail out if we hit the limit.
-  if (Depth++ >= MaxDepth)
-    return false;
 
   // Check for recursive pointer simplifications.
   if (V->getType()->isPointerTy()) {
     if (isKnownNonNullFromDominatingCondition(V, Q.CxtI, Q.DT))
       return true;
 
     if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V))
       if (isGEPKnownNonNull(GEP, Depth, Q))

test/Transforms/CorrelatedValuePropagation/pointer.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S -mtriple=i686-- -o - %s | FileCheck %s
+
+; Testcast that check that we don't end in a neverending recursion resulting in
+; a segmentation fault. The checks below verify that nothing is changed.
+
+declare dso_local i16* @f2(i16* readnone returned) local_unnamed_addr
+
+define dso_local void @f3() local_unnamed_addr {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[CALL6:%.*]] = call i16* @f2(i16* [[CALL6]])
+; CHECK-NEXT:    br i1 false, label [[FOR_COND]], label [[FOR_COND3:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[C_0:%.*]] = phi i16* [ undef, [[ENTRY:%.*]] ], [ [[CALL6]], [[FOR_END:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_COND3]]
+; CHECK:       for.cond3:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.end:
+  %call6 = call i16* @f2(i16* %call6)
+  br i1 false, label %for.cond, label %for.cond3
+
+for.cond:
+  %c.0 = phi i16* [ undef, %entry ], [ %call6, %for.end ]
+  br label %for.cond3
+
+for.cond3:
+  ret void
+}