[X86] Prevent fast isel from folding loads into the instructions listed in hasPartialRegUpdate.

[craig.topper]

Summary

This patch moves the check for opt size and hasPartialRegUpdate into the lower level implementation of foldMemoryOperandImpl to catch the entry point that fast isel uses.

We're still folding undef register instructions in AVX that we should also probably disable, but that's a problem for another patch.

Unfortunately, this requires reordering a bunch of functions which is why the diff is so large. I can do the function reordering separately if we want.

[RKSimon]

An NFC re-ordering of functions would make this a lot easier to see the relevant functional diffs.

[craig.topper]

Rebase after function reordering.

[zvi]

LGTM, with a minor concern that with this change FastISel is a bit slower because the bail-out condition was moved to a later point?

[RKSimon]

Do we have optsize/minsize tests for this anywhere?

[craig.topper]

I don't think this makes fast isel slower rather it make sure the fast isel entry point does this check. (at least the one used by X86FastISel::tryToFoldLoadIntoMI). So it actually may slow down peephole folding and spill folding. Maybe I should just leave the other checks in place and add a new one?

I'm not sure what tests we have for the optsize. I'm pretty sure we don't have any for fast isel but not sure about SDAG.

[craig.topper]

Replicate check instead of moving it. Add optsize fast-isel tests.

[RKSimon]

LGTM - do we have a TODO/bug anywhere for the existing AVX issue?

llvm/trunk/lib/Target/X86/X86InstrInfo.cpp

MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
   // do not fold loads into calls or pushes, unless optimizing for size
   // aggressively.
   if (isSlowTwoMemOps && !MF.getFunction()->optForMinSize() &&
       (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r ||
        MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r ||
        MI.getOpcode() == X86::PUSH64r))
     return nullptr;
 
+  // Avoid partial register update stalls unless optimizing for size.
+  // TODO: we should block undef reg update as well.
+  if (!MF.getFunction()->optForSize() && hasPartialRegUpdate(MI.getOpcode()))
+    return nullptr;
+
   unsigned NumOps = MI.getDesc().getNumOperands();
   bool isTwoAddr =
       NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1;
 
   // FIXME: AsmPrinter doesn't know how to handle
   // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding.
   if (MI.getOpcode() == X86::ADD32ri &&
       MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS)
X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
                                     MachineBasicBlock::iterator InsertPt,
                                     int FrameIndex, LiveIntervals *LIS) const {
   // Check switch flag
   if (NoFusing)
     return nullptr;
 
   // Unless optimizing for size, don't fold to avoid partial
   // register update stalls
+  // TODO: we should block undef reg update as well.
   if (!MF.getFunction()->optForSize() && hasPartialRegUpdate(MI.getOpcode()))
     return nullptr;
 
   // Don't fold subreg spills, or reloads that use a high subreg.
   for (auto Op : Ops) {
     MachineOperand &MO = MI.getOperand(Op);
     auto SubReg = MO.getSubReg();
     if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi))
if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF))
       return nullptr;
     return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS);
   }
 
   // Check switch flag
   if (NoFusing) return nullptr;
 
   // Avoid partial register update stalls unless optimizing for size.
+  // TODO: we should block undef reg update as well.
   if (!MF.getFunction()->optForSize() && hasPartialRegUpdate(MI.getOpcode()))
     return nullptr;
 
   // Determine the alignment of the load.
   unsigned Alignment = 0;
   if (LoadMI.hasOneMemOperand())
     Alignment = (*LoadMI.memoperands_begin())->getAlignment();
   else

llvm/trunk/test/CodeGen/X86/fast-isel-fptrunc-fpext.ll

 entry:
   %conv = fptrunc double %x to float
   ret float %conv
 }
 
 define double @single_to_double_rm(float* %x) {
 ; SSE-LABEL: single_to_double_rm:
 ; SSE:       # BB#0: # %entry
-; SSE-NEXT:    cvtss2sd (%rdi), %xmm0
+; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE-NEXT:    cvtss2sd %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: single_to_double_rm:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load float, float* %x, align 4
   %conv = fpext float %0 to double
   ret double %conv
 }
 
+define double @single_to_double_rm_optsize(float* %x) optsize {
+; SSE-LABEL: single_to_double_rm_optsize:
+; SSE:       # BB#0: # %entry
+; SSE-NEXT:    cvtss2sd (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: single_to_double_rm_optsize:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %0 = load float, float* %x, align 4
+  %conv = fpext float %0 to double
+  ret double %conv
+}
+
 define float @double_to_single_rm(double* %x) {
 ; SSE-LABEL: double_to_single_rm:
 ; SSE:       # BB#0: # %entry
-; SSE-NEXT:    cvtsd2ss (%rdi), %xmm0
+; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE-NEXT:    cvtsd2ss %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: double_to_single_rm:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load double, double* %x, align 8
   %conv = fptrunc double %0 to float
   ret float %conv
 }
+
+define float @double_to_single_rm_optsize(double* %x) optsize {
+; SSE-LABEL: double_to_single_rm_optsize:
+; SSE:       # BB#0: # %entry
+; SSE-NEXT:    cvtsd2ss (%rdi), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: double_to_single_rm_optsize:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %0 = load double, double* %x, align 8
+  %conv = fptrunc double %0 to float
+  ret float %conv
+}

llvm/trunk/test/CodeGen/X86/fast-isel-int-float-conversion-x86-64.ll

 entry:
   %0 = sitofp i64 %a to double
   ret double %0
 }
 
 define double @long_to_double_rm(i64* %a) {
 ; SSE2-LABEL: long_to_double_rm:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    cvtsi2sdq (%rdi), %xmm0
+; SSE2-NEXT:    movq (%rdi), %rax
+; SSE2-NEXT:    cvtsi2sdq %rax, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: long_to_double_rm:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load i64, i64* %a
   %1 = sitofp i64 %0 to double
   ret double %1
 }
 
+define double @long_to_double_rm_optsize(i64* %a) optsize {
+; SSE2-LABEL: long_to_double_rm_optsize:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    cvtsi2sdq (%rdi), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: long_to_double_rm_optsize:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %0 = load i64, i64* %a
+  %1 = sitofp i64 %0 to double
+  ret double %1
+}
+
 define float @long_to_float_rr(i64 %a) {
 ; SSE2-LABEL: long_to_float_rr:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    cvtsi2ssq %rdi, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: long_to_float_rr:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = sitofp i64 %a to float
   ret float %0
 }
 
 define float @long_to_float_rm(i64* %a) {
 ; SSE2-LABEL: long_to_float_rm:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    cvtsi2ssq (%rdi), %xmm0
+; SSE2-NEXT:    movq (%rdi), %rax
+; SSE2-NEXT:    cvtsi2ssq %rax, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: long_to_float_rm:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 entry:
   %0 = load i64, i64* %a
   %1 = sitofp i64 %0 to float
   ret float %1
 }
+
+define float @long_to_float_rm_optsize(i64* %a) optsize {
+; SSE2-LABEL: long_to_float_rm_optsize:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    cvtsi2ssq (%rdi), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: long_to_float_rm_optsize:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+entry:
+  %0 = load i64, i64* %a
+  %1 = sitofp i64 %0 to float
+  ret float %1
+}

llvm/trunk/test/CodeGen/X86/fast-isel-int-float-conversion.ll

 entry:
   %0 = sitofp i32 %a to double
   ret double %0
 }
 
 define double @int_to_double_rm(i32* %a) {
 ; SSE2-LABEL: int_to_double_rm:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    cvtsi2sdl (%rdi), %xmm0
+; SSE2-NEXT:    movl (%rdi), %eax
+; SSE2-NEXT:    cvtsi2sdl %eax, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: int_to_double_rm:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_double_rm:

 ; AVX_X86-NEXT:    .cfi_def_cfa %esp, 4
 ; AVX_X86-NEXT:    retl
 entry:
   %0 = load i32, i32* %a
   %1 = sitofp i32 %0 to double
   ret double %1
 }
 
+define double @int_to_double_rm_optsize(i32* %a) optsize {
+; SSE2-LABEL: int_to_double_rm_optsize:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    cvtsi2sdl (%rdi), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: int_to_double_rm_optsize:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; SSE2_X86-LABEL: int_to_double_rm_optsize:
+; SSE2_X86:       # BB#0: # %entry
+; SSE2_X86-NEXT:    pushl %ebp
+; SSE2_X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE2_X86-NEXT:    .cfi_offset %ebp, -8
+; SSE2_X86-NEXT:    movl %esp, %ebp
+; SSE2_X86-NEXT:    .cfi_def_cfa_register %ebp
+; SSE2_X86-NEXT:    andl $-8, %esp
+; SSE2_X86-NEXT:    subl $8, %esp
+; SSE2_X86-NEXT:    movl 8(%ebp), %eax
+; SSE2_X86-NEXT:    cvtsi2sdl (%eax), %xmm0
+; SSE2_X86-NEXT:    movsd %xmm0, (%esp)
+; SSE2_X86-NEXT:    fldl (%esp)
+; SSE2_X86-NEXT:    movl %ebp, %esp
+; SSE2_X86-NEXT:    popl %ebp
+; SSE2_X86-NEXT:    .cfi_def_cfa %esp, 4
+; SSE2_X86-NEXT:    retl
+;
+; AVX_X86-LABEL: int_to_double_rm_optsize:
+; AVX_X86:       # BB#0: # %entry
+; AVX_X86-NEXT:    pushl %ebp
+; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX_X86-NEXT:    .cfi_offset %ebp, -8
+; AVX_X86-NEXT:    movl %esp, %ebp
+; AVX_X86-NEXT:    .cfi_def_cfa_register %ebp
+; AVX_X86-NEXT:    andl $-8, %esp
+; AVX_X86-NEXT:    subl $8, %esp
+; AVX_X86-NEXT:    movl 8(%ebp), %eax
+; AVX_X86-NEXT:    vcvtsi2sdl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vmovsd %xmm0, (%esp)
+; AVX_X86-NEXT:    fldl (%esp)
+; AVX_X86-NEXT:    movl %ebp, %esp
+; AVX_X86-NEXT:    popl %ebp
+; AVX_X86-NEXT:    .cfi_def_cfa %esp, 4
+; AVX_X86-NEXT:    retl
+entry:
+  %0 = load i32, i32* %a
+  %1 = sitofp i32 %0 to double
+  ret double %1
+}
+
 define float @int_to_float_rr(i32 %a) {
 ; SSE2-LABEL: int_to_float_rr:
 ; SSE2:       # BB#0: # %entry
 ; SSE2-NEXT:    cvtsi2ssl %edi, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: int_to_float_rr:
 ; AVX:       # BB#0: # %entry

 entry:
   %0 = sitofp i32 %a to float
   ret float %0
 }
 
 define float @int_to_float_rm(i32* %a) {
 ; SSE2-LABEL: int_to_float_rm:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    cvtsi2ssl (%rdi), %xmm0
+; SSE2-NEXT:    movl (%rdi), %eax
+; SSE2-NEXT:    cvtsi2ssl %eax, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: int_to_float_rm:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; SSE2_X86-LABEL: int_to_float_rm:

 ; AVX_X86-NEXT:    popl %eax
 ; AVX_X86-NEXT:    .cfi_def_cfa_offset 4
 ; AVX_X86-NEXT:    retl
 entry:
   %0 = load i32, i32* %a
   %1 = sitofp i32 %0 to float
   ret float %1
 }
+
+define float @int_to_float_rm_optsize(i32* %a) optsize {
+; SSE2-LABEL: int_to_float_rm_optsize:
+; SSE2:       # BB#0: # %entry
+; SSE2-NEXT:    cvtsi2ssl (%rdi), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: int_to_float_rm_optsize:
+; AVX:       # BB#0: # %entry
+; AVX-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; SSE2_X86-LABEL: int_to_float_rm_optsize:
+; SSE2_X86:       # BB#0: # %entry
+; SSE2_X86-NEXT:    pushl %eax
+; SSE2_X86-NEXT:    .cfi_def_cfa_offset 8
+; SSE2_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SSE2_X86-NEXT:    cvtsi2ssl (%eax), %xmm0
+; SSE2_X86-NEXT:    movss %xmm0, (%esp)
+; SSE2_X86-NEXT:    flds (%esp)
+; SSE2_X86-NEXT:    popl %eax
+; SSE2_X86-NEXT:    .cfi_def_cfa_offset 4
+; SSE2_X86-NEXT:    retl
+;
+; AVX_X86-LABEL: int_to_float_rm_optsize:
+; AVX_X86:       # BB#0: # %entry
+; AVX_X86-NEXT:    pushl %eax
+; AVX_X86-NEXT:    .cfi_def_cfa_offset 8
+; AVX_X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX_X86-NEXT:    vcvtsi2ssl (%eax), %xmm0, %xmm0
+; AVX_X86-NEXT:    vmovss %xmm0, (%esp)
+; AVX_X86-NEXT:    flds (%esp)
+; AVX_X86-NEXT:    popl %eax
+; AVX_X86-NEXT:    .cfi_def_cfa_offset 4
+; AVX_X86-NEXT:    retl
+entry:
+  %0 = load i32, i32* %a
+  %1 = sitofp i32 %0 to float
+  ret float %1
+}