Fix another infinite loop in InstCombine

[steven_wu]

Summary

InstCombine infinite-loops for the testcase added
It is because InstCombine is generating instructions that can be
optimized by itself. Fix by not optimizing frem if the optimized
type is the same as original type.
rdar://problem/19150820

[majnemer]

LGTM with tweaks

test/Transforms/InstCombine/fpcast.ll

84–86 ↗

(On Diff #17209)

this should be:
; CHECK-NEXT: %[[frem:.*]] = frem double %[[fext]], 1.000000e-01
; CHECK-NEXT: %[[trunc:.*]] = fptrunc double %[[frem]] to float
; CHECK-NEXT: ret float %[[trunc]]

[steven_wu]

Sorry I forget to add llvm-commits for the first try. Since David has already reviewed it, I will not create a new review.

Steven

llvm/trunk/lib/Transforms/InstCombine/InstCombineCasts.cpp

switch (OpI->getOpcode()) {
           return RI;
         }
         break;
       case Instruction::FRem:
         // Remainder is straightforward.  Remainder is always exact, so the
         // type of OpI doesn't enter into things at all.  We simply evaluate
         // in whichever source type is larger, then convert to the
         // destination type.
+        if (SrcWidth != OpWidth) {
           if (LHSWidth < SrcWidth)
             LHSOrig = Builder->CreateFPExt(LHSOrig, RHSOrig->getType());
           else if (RHSWidth <= SrcWidth)
             RHSOrig = Builder->CreateFPExt(RHSOrig, LHSOrig->getType());
           if (LHSOrig != OpI->getOperand(0) || RHSOrig != OpI->getOperand(1)) {
             Value *ExactResult = Builder->CreateFRem(LHSOrig, RHSOrig);
             if (Instruction *RI = dyn_cast<Instruction>(ExactResult))
               RI->copyFastMathFlags(OpI);
             return CastInst::CreateFPCast(ExactResult, CI.getType());
           }
         }
+        break;
+    }
 
     // (fptrunc (fneg x)) -> (fneg (fptrunc x))
     if (BinaryOperator::isFNeg(OpI)) {
       Value *InnerTrunc = Builder->CreateFPTrunc(OpI->getOperand(1),
                                                  CI.getType());
       Instruction *RI = BinaryOperator::CreateFNeg(InnerTrunc);
       RI->copyFastMathFlags(OpI);
       return RI;

llvm/trunk/test/Transforms/InstCombine/fpcast.ll

define float @test7(double %V) {
   %frem = frem double %V, 1.000000e+00
   %trunc = fptrunc double %frem to float
   ret float %trunc
 ; CHECK-LABEL: @test7
 ; CHECK-NEXT: %[[frem:.*]]  = frem double %V, 1.000000e+00
 ; CHECK-NEXT: %[[trunc:.*]] = fptrunc double %frem to float
 ; CHECK-NEXT: ret float %trunc
 }
+
+define float @test8(float %V) {
+  %fext = fpext float %V to double
+  %frem = frem double %fext, 1.000000e-01
+  %trunc = fptrunc double %frem to float
+  ret float %trunc
+; CHECK-LABEL: @test8
+; CHECK-NEXT: %[[fext:.*]]  = fpext float %V to double
+; CHECK-NEXT: %[[frem:.*]]  = frem double %fext, 1.000000e-01
+; CHECK-NEXT: %[[trunc:.*]] = fptrunc double %frem to float
+; CHECK-NEXT: ret float %trunc
+}