[X86] Add i32->float and i64->double bitcast pseudo instructions to store folding table.

[craig.topper]

Summary

We have pseudo instructions we use for bitcasts between these types.
We have them in the load folding table, but not the store folding
table. This adds them there so they can be used for stack spills.

I added an exact size check so that we don't fold when the stack slot
is larger than the GPR. Otherwise the upper bits in the stack slot
would be garbage. That would be fine for Eli's test case in PR47874,
but I'm not sure its safe in general.

A step towards fixing PR47874. Next steps are to change the ADDSSrr_Int
pseudo instructions to use FR32 as the second source register class
instead of VR128. That will keep the coalescer from promoting the
register class of the bitcast instruction which will make the stack
slot 4 bytes instead of 16 bytes.

[RKSimon]

LGTM

llvm/lib/Target/X86/X86InstrInfo.cpp

5529

Add a comment here or down below explaining the purpose + limits on reg/store size.

llvm/lib/Target/X86/X86InstrFoldTables.cpp

 //===-- X86InstrFoldTables.cpp - X86 Instruction Folding Tables -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
static const X86MemoryFoldTableEntry MemoryFoldTable2Addr[] = {
   { X86::XOR64ri8,    X86::XOR64mi8,   0 },
   { X86::XOR64rr,     X86::XOR64mr,    0 },
   { X86::XOR8ri,      X86::XOR8mi,     0 },
   { X86::XOR8ri8,     X86::XOR8mi8,    0 },
   { X86::XOR8rr,      X86::XOR8mr,     0 },
 };
 
 static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
   { X86::BT16ri8,             X86::BT16mi8,             TB_FOLDED_LOAD },
   { X86::BT32ri8,             X86::BT32mi8,             TB_FOLDED_LOAD },
   { X86::BT64ri8,             X86::BT64mi8,             TB_FOLDED_LOAD },
   { X86::CALL16r,             X86::CALL16m,             TB_FOLDED_LOAD },
   { X86::CALL16r_NT,          X86::CALL16m_NT,          TB_FOLDED_LOAD },
   { X86::CALL32r,             X86::CALL32m,             TB_FOLDED_LOAD },
   { X86::CALL32r_NT,          X86::CALL32m_NT,          TB_FOLDED_LOAD },
   { X86::CALL64r,             X86::CALL64m,             TB_FOLDED_LOAD },
   { X86::CALL64r_NT,          X86::CALL64m_NT,          TB_FOLDED_LOAD },
static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
   { X86::MMX_MOVD64from64rr,  X86::MMX_MOVD64from64rm,  TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MMX_MOVD64grr,       X86::MMX_MOVD64mr,        TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MOV16ri,             X86::MOV16mi,             TB_FOLDED_STORE },
   { X86::MOV16rr,             X86::MOV16mr,             TB_FOLDED_STORE },
   { X86::MOV32ri,             X86::MOV32mi,             TB_FOLDED_STORE },
   { X86::MOV32rr,             X86::MOV32mr,             TB_FOLDED_STORE },
   { X86::MOV64ri32,           X86::MOV64mi32,           TB_FOLDED_STORE },
   { X86::MOV64rr,             X86::MOV64mr,             TB_FOLDED_STORE },
+  { X86::MOV64toSDrr,         X86::MOV64mr,             TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MOV8ri,              X86::MOV8mi,              TB_FOLDED_STORE },
   { X86::MOV8rr,              X86::MOV8mr,              TB_FOLDED_STORE },
   { X86::MOV8rr_NOREX,        X86::MOV8mr_NOREX,        TB_FOLDED_STORE },
   { X86::MOVAPDrr,            X86::MOVAPDmr,            TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::MOVAPSrr,            X86::MOVAPSmr,            TB_FOLDED_STORE | TB_ALIGN_16 },
+  { X86::MOVDI2SSrr,          X86::MOV32mr,             TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MOVDQArr,            X86::MOVDQAmr,            TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::MOVDQUrr,            X86::MOVDQUmr,            TB_FOLDED_STORE },
   { X86::MOVPDI2DIrr,         X86::MOVPDI2DImr,         TB_FOLDED_STORE },
   { X86::MOVPQIto64rr,        X86::MOVPQI2QImr,         TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MOVSDto64rr,         X86::MOVSDmr,             TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MOVSS2DIrr,          X86::MOVSSmr,             TB_FOLDED_STORE },
   { X86::MOVUPDrr,            X86::MOVUPDmr,            TB_FOLDED_STORE },
   { X86::MOVUPSrr,            X86::MOVUPSmr,            TB_FOLDED_STORE },
static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
   { X86::VEXTRACTI32x4Z256rr, X86::VEXTRACTI32x4Z256mr, TB_FOLDED_STORE },
   { X86::VEXTRACTI32x4Zrr,    X86::VEXTRACTI32x4Zmr,    TB_FOLDED_STORE },
   { X86::VEXTRACTI32x8Zrr,    X86::VEXTRACTI32x8Zmr,    TB_FOLDED_STORE },
   { X86::VEXTRACTI64x2Z256rr, X86::VEXTRACTI64x2Z256mr, TB_FOLDED_STORE },
   { X86::VEXTRACTI64x2Zrr,    X86::VEXTRACTI64x2Zmr,    TB_FOLDED_STORE },
   { X86::VEXTRACTI64x4Zrr,    X86::VEXTRACTI64x4Zmr,    TB_FOLDED_STORE },
   { X86::VEXTRACTPSZrr,       X86::VEXTRACTPSZmr,       TB_FOLDED_STORE },
   { X86::VEXTRACTPSrr,        X86::VEXTRACTPSmr,        TB_FOLDED_STORE },
+  { X86::VMOV64toSDZrr,       X86::MOV64mr,             TB_FOLDED_STORE | TB_NO_REVERSE },
+  { X86::VMOV64toSDrr,        X86::MOV64mr,             TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::VMOVAPDYrr,          X86::VMOVAPDYmr,          TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVAPDZ128rr,       X86::VMOVAPDZ128mr,       TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::VMOVAPDZ256rr,       X86::VMOVAPDZ256mr,       TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVAPDZrr,          X86::VMOVAPDZmr,          TB_FOLDED_STORE | TB_ALIGN_64 },
   { X86::VMOVAPDrr,           X86::VMOVAPDmr,           TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::VMOVAPSYrr,          X86::VMOVAPSYmr,          TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVAPSZ128rr,       X86::VMOVAPSZ128mr,       TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::VMOVAPSZ256rr,       X86::VMOVAPSZ256mr,       TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVAPSZrr,          X86::VMOVAPSZmr,          TB_FOLDED_STORE | TB_ALIGN_64 },
   { X86::VMOVAPSrr,           X86::VMOVAPSmr,           TB_FOLDED_STORE | TB_ALIGN_16 },
+  { X86::VMOVDI2SSZrr,        X86::MOV32mr,             TB_FOLDED_STORE | TB_NO_REVERSE },
+  { X86::VMOVDI2SSrr,         X86::MOV32mr,             TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::VMOVDQA32Z128rr,     X86::VMOVDQA32Z128mr,     TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::VMOVDQA32Z256rr,     X86::VMOVDQA32Z256mr,     TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVDQA32Zrr,        X86::VMOVDQA32Zmr,        TB_FOLDED_STORE | TB_ALIGN_64 },
   { X86::VMOVDQA64Z128rr,     X86::VMOVDQA64Z128mr,     TB_FOLDED_STORE | TB_ALIGN_16 },
   { X86::VMOVDQA64Z256rr,     X86::VMOVDQA64Z256mr,     TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVDQA64Zrr,        X86::VMOVDQA64Zmr,        TB_FOLDED_STORE | TB_ALIGN_64 },
   { X86::VMOVDQAYrr,          X86::VMOVDQAYmr,          TB_FOLDED_STORE | TB_ALIGN_32 },
   { X86::VMOVDQArr,           X86::VMOVDQAmr,           TB_FOLDED_STORE | TB_ALIGN_16 },

llvm/lib/Target/X86/X86InstrInfo.cpp

 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
if (OpNum == 0) {
       }
     }
 
     I = lookupFoldTable(MI.getOpcode(), OpNum);
   }
 
   if (I != nullptr) {
     unsigned Opcode = I->DstOp;
+    bool FoldedLoad =
+        isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_LOAD) || OpNum > 0;
+    bool FoldedStore =
+        isTwoAddrFold || (OpNum == 0 && I->Flags & TB_FOLDED_STORE);
     MaybeAlign MinAlign =
         decodeMaybeAlign((I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT);
     if (MinAlign && Alignment < *MinAlign)
       return nullptr;
     bool NarrowToMOV32rm = false;
     if (Size) {
       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
       const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum,
                                                   &RI, MF);
       unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8;
-      if (Size < RCSize) {
+      if (FoldedLoad && Size < RCSize) {
         // FIXME: Allow scalar intrinsic instructions like ADDSSrm_Int.
         // Check if it's safe to fold the load. If the size of the object is
         // narrower than the load width, then it's not.
         if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4)
           return nullptr;
         // If this is a 64-bit load, but the spill slot is 32, then we can do
         // a 32-bit load which is implicitly zero-extended. This likely is
         // due to live interval analysis remat'ing a load from stack slot.
         if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
           return nullptr;
         Opcode = X86::MOV32rm;
         NarrowToMOV32rm = true;
       }
+      if (FoldedStore && Size != RCSize)
+        return nullptr;
     }
 
     if (isTwoAddrFold)
       NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this);
     else
       NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this);
 
     if (NarrowToMOV32rm) {

llvm/test/CodeGen/X86/pr47874.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin19.6.0 | FileCheck %s --check-prefixes=CHECK,SSE2
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin19.6.0 -mattr=avx | FileCheck %s --check-prefixes=CHECK,AVX
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin19.6.0 -mattr=avx512f | FileCheck %s --check-prefixes=CHECK,AVX
 
 define void @a(float* %arg, i32 %arg1) {
 ; SSE2-LABEL: a:
 ; SSE2:       ## %bb.0: ## %bb
 ; SSE2-NEXT:    testl %esi, %esi
 ; SSE2-NEXT:    jle LBB0_3
 ; SSE2-NEXT:  ## %bb.1: ## %bb2
-; SSE2-NEXT:    movd %esi, %xmm0
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
+; SSE2-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; SSE2-NEXT:    movl %esi, %eax
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  LBB0_2: ## %bb6
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    ## InlineAsm Start
 ; SSE2-NEXT:    ## InlineAsm End
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    addss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 4-byte Folded Reload
 ; SSE2-NEXT:    movss %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $4, %rdi
 ; SSE2-NEXT:    decq %rax
 ; SSE2-NEXT:    jne LBB0_2
 ; SSE2-NEXT:  LBB0_3: ## %bb5
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: a:
 ; AVX:       ## %bb.0: ## %bb
 ; AVX-NEXT:    testl %esi, %esi
 ; AVX-NEXT:    jle LBB0_3
 ; AVX-NEXT:  ## %bb.1: ## %bb2
-; AVX-NEXT:    vmovd %esi, %xmm0
-; AVX-NEXT:    vmovd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Folded Spill
+; AVX-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; AVX-NEXT:    movl %esi, %eax
 ; AVX-NEXT:    .p2align 4, 0x90
 ; AVX-NEXT:  LBB0_2: ## %bb6
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    ## InlineAsm Start
 ; AVX-NEXT:    ## InlineAsm End
 ; AVX-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 4-byte Reload
 ; AVX-NEXT:    ## xmm0 = mem[0],zero,zero,zero

 }
 
 define void @b(double* %arg, i64 %arg1) {
 ; SSE2-LABEL: b:
 ; SSE2:       ## %bb.0: ## %bb
 ; SSE2-NEXT:    testq %rsi, %rsi
 ; SSE2-NEXT:    jle LBB1_3
 ; SSE2-NEXT:  ## %bb.1: ## %bb2
-; SSE2-NEXT:    movq %rsi, %xmm0
-; SSE2-NEXT:    movq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Folded Spill
+; SSE2-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  LBB1_2: ## %bb6
 ; SSE2-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    ## InlineAsm Start
 ; SSE2-NEXT:    ## InlineAsm End
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE2-NEXT:    addsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 8-byte Folded Reload
 ; SSE2-NEXT:    movsd %xmm0, (%rdi)
 ; SSE2-NEXT:    addq $8, %rdi
 ; SSE2-NEXT:    decq %rsi
 ; SSE2-NEXT:    jne LBB1_2
 ; SSE2-NEXT:  LBB1_3: ## %bb5
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: b:
 ; AVX:       ## %bb.0: ## %bb
 ; AVX-NEXT:    testq %rsi, %rsi
 ; AVX-NEXT:    jle LBB1_3
 ; AVX-NEXT:  ## %bb.1: ## %bb2
-; AVX-NEXT:    vmovq %rsi, %xmm0
-; AVX-NEXT:    vmovq %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Folded Spill
+; AVX-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; AVX-NEXT:    .p2align 4, 0x90
 ; AVX-NEXT:  LBB1_2: ## %bb6
 ; AVX-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    ## InlineAsm Start
 ; AVX-NEXT:    ## InlineAsm End
 ; AVX-NEXT:    vmovsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 8-byte Reload
 ; AVX-NEXT:    ## xmm0 = mem[0],zero
 ; AVX-NEXT:    vaddsd (%rdi), %xmm0, %xmm0