[CostModel] Fix for cast crash

[samparker]

Summary

Don't assume that the 'NextLevelOp' is an instruction.

Bugzilla: https://bugs.llvm.org/show_bug.cgi?id=46430

[dfukalov]

llvm/lib/Analysis/TargetTransformInfo.cpp

1055–1062

It seems in case !isa<Instruction>(NextLevelOp) we should return TTI::RK_None just like for !NextLevelRD case. Correct me if I'm wrong.

1079–1082

You can just replace cast to dyn_cast here since this recursive call will return TTI::RK_None for nullptr as first parameter.

[uabelho]

I don't know this code but at least the crash I saw goes away with the fix. Thanks!

[samparker]

Updated according to @dfukalov suggestions.

[dfukalov]

llvm/lib/Analysis/TargetTransformInfo.cpp

1081

unreachable return typo )

[samparker]

Thanks, fixed typo.

llvm/lib/Analysis/TargetTransformInfo.cpp

else if (NextLevelOpR && NextLevelOpR != RD->LHS)
       return TTI::RK_None;
 
     NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS;
   } else
     return TTI::RK_None;
 
   // Check that the next levels binary operation exists and matches with the
   // current one.
   if (Level + 1 != NumLevels) {
+    if (!isa<Instruction>(NextLevelOp))
+      return TTI::RK_None;
     Optional<TTI::ReductionData> NextLevelRD =
         getReductionData(cast<Instruction>(NextLevelOp));
     if (!NextLevelRD || !RD->hasSameData(*NextLevelRD))
       return TTI::RK_None;
   }
 
   // Shuffle mask for pairwise operation must match.
   if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) {
     if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level))
       return TTI::RK_None;
   } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) {
     if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level))
       return TTI::RK_None;
   } else {
     return TTI::RK_None;
   }
 
   if (++Level == NumLevels)
     return RD->Kind;
 
   // Match next level.
-  return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level,
+  return matchPairwiseReductionAtLevel(dyn_cast<Instruction>(NextLevelOp), Level,
                                        NumLevels);
 }
 
 TTI::ReductionKind TTI::matchPairwiseReduction(
   const ExtractElementInst *ReduxRoot, unsigned &Opcode, VectorType *&Ty) {
   if (!EnableReduxCost)
     return TTI::RK_None;
 
   // Need to extract the first element.
   ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
   unsigned Idx = ~0u;

llvm/test/Transforms/LoopUnroll/X86/pr46430.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -loop-unroll -costmodel-reduxcost -S -o - %s | FileCheck %s
+
+define void @g() {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[F_EXIT:%.*]]
+; CHECK:       f.exit:
+; CHECK-NEXT:    [[RDX_SHUF9_I:%.*]] = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX10_I:%.*]] = xor <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, [[RDX_SHUF9_I]]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[BIN_RDX10_I]], i32 0
+; CHECK-NEXT:    br label [[F_EXIT]]
+;
+entry:
+  br label %f.exit
+
+f.exit:                                           ; preds = %f.exit, %entry
+  %rdx.shuf9.i = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx10.i = xor <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, %rdx.shuf9.i
+  %0 = extractelement <4 x i16> %bin.rdx10.i, i32 0
+  br label %f.exit
+}
+