[X86][FPEnv] Lowering of {get,set,reset}_fpenv

[sepavloff]

Summary

The change implements lowering of get_fpenv, set_fpenv and
reset_fpenv.

[sepavloff]

Fixed formatting issues

[craig.topper]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

390

Windows defaults to 53 bit precision.

[sepavloff]

Updated patch after update of dependency

[sepavloff]

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

390

Fixed.

[sepavloff]

Updated patch

• Remove changes related to fix for mayLoad/mayStore properties, the relevant changes are already in master,
• Removed lowering of ISD::GET_FPENV and friends. This operation is useless as lowering of FPEnv intrinsics is made when constructing DAG.

[sepavloff]

Intrinsics now do not use pointers

[sepavloff]

Update after update of dependency

[sepavloff]

Rebase and ping

[sepavloff]

Ping.

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

26122

Maybe check for hasX87 too?

26126

hasSSE1?

26158

Maybe pass Subtarget from caller instead?

llvm/lib/Target/X86/X86ISelLowering.h

778

Add a suffix m

781

ditto.

llvm/lib/Target/X86/X86InstrFPStack.td

400–401

Need to add Predicates = [HasX87] if change to I.

406–407

ditto.

[sepavloff]

Fix the patch according to reviewer's comments

[pengfei]

LGTM.

llvm/lib/Target/X86/X86ISelLowering.cpp

29165–29167

Maybe use upper L to match with other functions here.

[sepavloff]

Thanks!

llvm/lib/Target/X86/X86ISelLowering.cpp

29165–29167

No problem.

llvm/lib/Target/X86/X86ISelLowering.h

enum NodeType : unsigned {
     VEXTRACT_STORE,
 
     // scalar broadcast from memory
     VBROADCAST_LOAD,
 
     // Store FP control world into i16 memory.
     FNSTCW16m,
 
+    // Store x87 FPU environment into memory.
+    FNSTENV,
+
+    // Load x87 FPU environment from memory.
+    FLDENV,
+
     /// This instruction implements FP_TO_SINT with the
     /// integer destination in memory and a FP reg source.  This corresponds
     /// to the X86::FIST*m instructions and the rounding mode change stuff. It
     /// has two inputs (token chain and address) and two outputs (int value
     /// and token chain). Memory VT specifies the type to store to.
     FP_TO_INT_IN_MEM,
 
     /// This instruction implements SINT_TO_FP with the

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
                                        SelectionDAG &DAG) {
   MVT VT = Op.getSimpleValueType();
   int NumElts = VT.getVectorNumElements();
   int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8);
   MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes);
 
   // Per-nibble leading zero PSHUFB lookup table.
   const int LUT[16] = {/* 0 */ 4, /* 1 */ 3, /* 2 */ 2, /* 3 */ 2,
                        /* 4 */ 1, /* 5 */ 1, /* 6 */ 1, /* 7 */ 1,
                        /* 8 */ 0, /* 9 */ 0, /* a */ 0, /* b */ 0,
                        /* c */ 0, /* d */ 0, /* e */ 0, /* f */ 0};
 
   SmallVector<SDValue, 64> LUTVec;
   for (int i = 0; i < NumBytes; ++i)
     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
   SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
 
   // Begin by bitcasting the input to byte vector, then split those bytes
   // into lo/hi nibbles and use the PSHUFB LUT to perform CLTZ on each of them.
   // If the hi input nibble is zero then we add both results together, otherwise
   // we just take the hi result (by masking the lo result to zero before the
static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
 
   Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
   Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
   Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
   SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
 
   // Merge result back from vXi8 back to VT, working on the lo/hi halves
   // of the current vector width in the same way we did for the nibbles.
   // If the upper half of the input element is zero then add the halves'
   // leading zero counts together, otherwise just use the upper half's.
   // Double the width of the result until we are at target width.
   while (CurrVT != VT) {
     int CurrScalarSizeInBits = CurrVT.getScalarSizeInBits();
     int CurrNumElts = CurrVT.getVectorNumElements();
     MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2);
     MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2);
     SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
   case ISD::EH_SJLJ_SETJMP:     return lowerEH_SJLJ_SETJMP(Op, DAG);
   case ISD::EH_SJLJ_LONGJMP:    return lowerEH_SJLJ_LONGJMP(Op, DAG);
   case ISD::EH_SJLJ_SETUP_DISPATCH:
     return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
   case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
   case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
   case ISD::FLT_ROUNDS_:        return LowerFLT_ROUNDS_(Op, DAG);
   case ISD::CTLZ:
   case ISD::CTLZ_ZERO_UNDEF:    return LowerCTLZ(Op, Subtarget, DAG);
   case ISD::CTTZ:
   case ISD::CTTZ_ZERO_UNDEF:    return LowerCTTZ(Op, Subtarget, DAG);
   case ISD::MUL:                return LowerMUL(Op, Subtarget, DAG);
   case ISD::MULHS:
   case ISD::MULHU:              return LowerMULH(Op, Subtarget, DAG);
   case ISD::ROTL:
   case ISD::ROTR:               return LowerRotate(Op, Subtarget, DAG);
   case ISD::SRA:
   case ISD::SRL:
#define NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE;
   NODE_NAME_CASE(TLSBASEADDR)
   NODE_NAME_CASE(TLSCALL)
   NODE_NAME_CASE(EH_SJLJ_SETJMP)
   NODE_NAME_CASE(EH_SJLJ_LONGJMP)
   NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH)
   NODE_NAME_CASE(EH_RETURN)
   NODE_NAME_CASE(TC_RETURN)
   NODE_NAME_CASE(FNSTCW16m)
+  NODE_NAME_CASE(FNSTENV)
+  NODE_NAME_CASE(FLDENV)
   NODE_NAME_CASE(LCMPXCHG_DAG)
   NODE_NAME_CASE(LCMPXCHG8_DAG)
   NODE_NAME_CASE(LCMPXCHG16_DAG)
   NODE_NAME_CASE(LCMPXCHG8_SAVE_EBX_DAG)
   NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG)
   NODE_NAME_CASE(LADD)
   NODE_NAME_CASE(LSUB)
   NODE_NAME_CASE(LOR)

llvm/lib/Target/X86/X86InstrFPStack.td

 
 def SDTX86Fld       : SDTypeProfile<1, 1, [SDTCisFP<0>,
                                            SDTCisPtrTy<1>]>;
 def SDTX86Fst       : SDTypeProfile<0, 2, [SDTCisFP<0>,
                                            SDTCisPtrTy<1>]>;
 def SDTX86Fild      : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisPtrTy<1>]>;
 def SDTX86Fist      : SDTypeProfile<0, 2, [SDTCisFP<0>, SDTCisPtrTy<1>]>;
 
-def SDTX86CwdStore  : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
+def SDTX86FPEnv     : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
 
 def X86fld          : SDNode<"X86ISD::FLD", SDTX86Fld,
                              [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def X86fst          : SDNode<"X86ISD::FST", SDTX86Fst,
                              [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def X86fild         : SDNode<"X86ISD::FILD", SDTX86Fild,
                              [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def X86fist         : SDNode<"X86ISD::FIST", SDTX86Fist,
                              [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def X86fp_to_mem : SDNode<"X86ISD::FP_TO_INT_IN_MEM", SDTX86Fst,
                           [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
-def X86fp_cwd_get16 : SDNode<"X86ISD::FNSTCW16m",          SDTX86CwdStore,
+def X86fp_cwd_get16 : SDNode<"X86ISD::FNSTCW16m",          SDTX86FPEnv,
                              [SDNPHasChain, SDNPMayStore, SDNPSideEffect,
                               SDNPMemOperand]>;
+def X86fpenv_get    : SDNode<"X86ISD::FNSTENV",            SDTX86FPEnv,
+                             [SDNPHasChain, SDNPMayStore, SDNPSideEffect,
+                              SDNPMemOperand]>;
+def X86fpenv_set    : SDNode<"X86ISD::FLDENV",             SDTX86FPEnv,
+                             [SDNPHasChain, SDNPMayLoad, SDNPSideEffect,
+                              SDNPMemOperand]>;
 
 def X86fstf32 : PatFrag<(ops node:$val, node:$ptr),
                         (X86fst node:$val, node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f32;
 }]>;
 def X86fstf64 : PatFrag<(ops node:$val, node:$ptr),
                         (X86fst node:$val, node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f64;

 def FICOM16m : FPI<0xDE, MRM2m, (outs), (ins i16mem:$src), "ficom{s}\t$src">;
 def FICOMP16m: FPI<0xDE, MRM3m, (outs), (ins i16mem:$src), "ficomp{s}\t$src">;
 
 def FICOM32m : FPI<0xDA, MRM2m, (outs), (ins i32mem:$src), "ficom{l}\t$src">;
 def FICOMP32m: FPI<0xDA, MRM3m, (outs), (ins i32mem:$src), "ficomp{l}\t$src">;
 } // SchedRW
 
 let SchedRW = [WriteMicrocoded] in {
 let Defs = [FPSW, FPCW], mayLoad = 1 in {
-def FLDENVm  : FPI<0xD9, MRM4m, (outs), (ins f32mem:$src), "fldenv\t$src">;
-def FRSTORm  : FPI<0xDD, MRM4m, (outs), (ins f32mem:$dst), "frstor\t$dst">;
+def FLDENVm  : I<0xD9, MRM4m, (outs), (ins f32mem:$src), "fldenv\t$src",
+                 [(X86fpenv_set addr:$src)]>;
+def FRSTORm  : I<0xDD, MRM4m, (outs), (ins f32mem:$dst), "frstor\t$dst", []>;
 }
 
 let Defs = [FPSW, FPCW], Uses = [FPSW, FPCW], mayStore = 1 in {
-def FSTENVm  : FPI<0xD9, MRM6m, (outs), (ins f32mem:$dst), "fnstenv\t$dst">;
-def FSAVEm   : FPI<0xDD, MRM6m, (outs), (ins f32mem:$dst), "fnsave\t$dst">;
+def FSTENVm  : I<0xD9, MRM6m, (outs), (ins f32mem:$dst), "fnstenv\t$dst",
+                 [(X86fpenv_get addr:$dst)]>;
+def FSAVEm   : I<0xDD, MRM6m, (outs), (ins f32mem:$dst), "fnsave\t$dst", []>;
 }
 
 let Uses = [FPSW], mayStore = 1 in
 def FNSTSWm  : FPI<0xDD, MRM7m, (outs), (ins i16mem:$dst), "fnstsw\t$dst">;
 
 let mayLoad = 1 in
 def FBLDm    : FPI<0xDF, MRM4m, (outs), (ins f80mem:$src), "fbld\t$src">;
 let Uses = [FPCW] ,mayRaiseFPException = 1, mayStore = 1 in

llvm/lib/Target/X86/X86SelectionDAGInfo.h

SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl,
                                   MachinePointerInfo DstPtrInfo) const override;
 
   SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                                   SDValue Chain, SDValue Dst, SDValue Src,
                                   SDValue Size, unsigned Align, bool isVolatile,
                                   bool AlwaysInline,
                                   MachinePointerInfo DstPtrInfo,
                                   MachinePointerInfo SrcPtrInfo) const override;
+
+  SDValue emitTargetCodeForGetFPEnv(SelectionDAG &DAG, const SDLoc &DL,
+                                    SDValue Chain, SDValue Addr,
+                                    MachinePointerInfo PtrInfo) const override;
+
+  SDValue emitTargetCodeForSetFPEnv(SelectionDAG &DAG, const SDLoc &DL,
+                                    SDValue Chain, SDValue Addr,
+                                    MachinePointerInfo PtrInfo) const override;
+
+  SDValue emitTargetCodeForResetFPEnv(SelectionDAG &DAG, const SDLoc &DL,
+                                      SDValue Chain) const override;
 };
 
 }
 
 #endif

llvm/lib/Target/X86/X86SelectionDAGInfo.cpp

 #include "X86ISelLowering.h"
 #include "X86InstrInfo.h"
 #include "X86RegisterInfo.h"
 #include "X86Subtarget.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/IntrinsicsX86.h"
+#include "llvm/IR/Module.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "x86-selectiondag-info"
 
 bool X86SelectionDAGInfo::isBaseRegConflictPossible(
     SelectionDAG &DAG, ArrayRef<MCPhysReg> ClobberSet) const {
   // We cannot use TRI->hasBasePointer() until *after* we select all basic
SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
   if (ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size))
     return emitConstantSizeRepmov(DAG, Subtarget, dl, Chain, Dst, Src,
                                   ConstantSize->getZExtValue(),
                                   Size.getValueType(), Align, isVolatile,
                                   AlwaysInline, DstPtrInfo, SrcPtrInfo);
 
   return SDValue();
 }
+
+// Size of X87 environment.
+const unsigned X87StateSize = 28;
+
+SDValue X86SelectionDAGInfo::emitTargetCodeForGetFPEnv(
+    SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Addr,
+    MachinePointerInfo PtrInfo) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+
+  // Get X87 state.
+  MachineMemOperand *X87EnvMMO = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOStore, X87StateSize, Align(4));
+  Chain =
+      DAG.getMemIntrinsicNode(X86ISD::FNSTENV, DL, DAG.getVTList(MVT::Other),
+                              {Chain, Addr}, MVT::i32, X87EnvMMO);
+  // FNSTENV changes the exception mask, so load back the stored environment.
+  Chain = DAG.getMemIntrinsicNode(X86ISD::FLDENV, DL, DAG.getVTList(MVT::Other),
+                                  {Chain, Addr}, MVT::i32, X87EnvMMO);
+
+  // If target supports SSE, get MXCSR as well.
+  if (DAG.getSubtarget().hasFeature(X86::FeatureSSE1)) {
+    SDValue MXCSRAddr =
+        DAG.getNode(ISD::ADD, DL, Addr.getValueType(), Addr,
+                    DAG.getConstant(X87StateSize, DL, Addr.getValueType()));
+    // Store MXCSR into memory.
+    Chain = DAG.getNode(
+        ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
+        DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
+        MXCSRAddr);
+  }
+
+  return Chain;
+}
+
+SDValue X86SelectionDAGInfo::emitTargetCodeForSetFPEnv(
+    SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Addr,
+    MachinePointerInfo PtrInfo) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+
+  // Get X87 state.
+  SDValue Ops[] = {Chain, Addr};
+  MachineMemOperand *X87EnvMMO = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOLoad, X87StateSize, Align(4));
+  Chain = DAG.getMemIntrinsicNode(X86ISD::FLDENV, DL, DAG.getVTList(MVT::Other),
+                                  Ops, MVT::i32, X87EnvMMO);
+
+  // If target supports SSE, set MXCSR as well.
+  if (DAG.getSubtarget().hasFeature(X86::FeatureSSE1)) {
+    // Get pointer to the MXCSR location in memory.
+    auto PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits());
+    SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Addr,
+                                    DAG.getConstant(X87StateSize, DL, PtrVT));
+    // Load MXCSR from memory.
+    Chain = DAG.getNode(
+        ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
+        DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
+        MXCSRAddr);
+  }
+
+  return Chain;
+}
+
+SDValue X86SelectionDAGInfo::emitTargetCodeForResetFPEnv(SelectionDAG &DAG,
+                                                         const SDLoc &DL,
+                                                         SDValue Chain) const {
+  const auto &Subtarget = static_cast<const X86Subtarget &>(DAG.getSubtarget());
+  IntegerType *ItemTy = Type::getInt32Ty(*DAG.getContext());
+  ArrayType *FPEnvTy = ArrayType::get(ItemTy, 8);
+  SmallVector<Constant *, 8> FPEnvVals;
+
+  // x87 FPU Control Word: mask all floating-point exceptions, sets rounding to
+  // nearest. FPU precision is set to 53 bits on Windows and 64 bits otherwise
+  // for compatibility with glibc.
+  unsigned X87CW = Subtarget.isTargetWindowsMSVC() ? 0x27F : 0x37F;
+  FPEnvVals.push_back(ConstantInt::get(ItemTy, X87CW));
+  Constant *Zero = ConstantInt::get(ItemTy, 0);
+  for (unsigned I = 0; I < 6; ++I)
+    FPEnvVals.push_back(Zero);
+
+  // MXCSR: mask all floating-point exceptions, sets rounding to nearest, clear
+  // all exceptions, sets DAZ and FTZ to 0.
+  FPEnvVals.push_back(ConstantInt::get(ItemTy, 0x1F80));
+  Constant *FPEnvBits = ConstantArray::get(FPEnvTy, FPEnvVals);
+  MVT PVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
+  SDValue CP = DAG.getConstantPool(FPEnvBits, PVT);
+  MachinePointerInfo MPI =
+      MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
+
+  return emitTargetCodeForSetFPEnv(DAG, DL, Chain, CP, MPI);
+}

llvm/test/CodeGen/X86/fpenv.ll

+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=-sse -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,X86-NOSSE
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+sse -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,X86-SSE
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,X64
+
+
+define void @func_01(i8* %fpenv) {
+entry:
+  call void @llvm.get.fpenv(i8* %fpenv)
+  ret void
+}
+; CHECK-LABEL: func_01:
+
+; X86-NOSSE:      fnstenv (%eax)
+; X86-NOSSE-NEXT: fldenv  (%eax)
+; X86-NOSSE-NEXT: retl
+
+; X86-SSE:        fnstenv (%eax)
+; X86-SSE-NEXT:   fldenv  (%eax)
+; X86-SSE-NEXT:   stmxcsr 28(%eax)
+; X86-SSE-NEXT:   retl
+
+; X64:            fnstenv (%rdi)
+; X64-NEXT:       fldenv  (%rdi)
+; X64-NEXT:       stmxcsr 28(%rdi)
+; X64-NEXT:       retq
+
+
+define void @func_02(i8* %fpenv) {
+entry:
+  call void @llvm.set.fpenv(i8* %fpenv)
+  ret void
+}
+; CHECK-LABEL: func_02:
+
+; X86-NOSSE:       fldenv (%eax)
+; X86-NOSSE-NEXT:  retl
+
+; X86-SSE:         fldenv  (%eax)
+; X86-SSE-NEXT:    ldmxcsr 28(%eax)
+; X86-SSE-NEXT:    retl
+
+; X64:             fldenv  (%rdi)
+; X64-NEXT:        ldmxcsr 28(%rdi)
+; X64-NEXT:        retq
+
+
+define void @func_03() {
+entry:
+  call void @llvm.reset.fpenv()
+  ret void
+}
+; CHECK: .LCPI{{.*}}:
+; CHECK:  .long 895                     # 0x37f
+; CHECK:  .long 0                       # 0x0
+; CHECK:  .long 0                       # 0x0
+; CHECK:  .long 0                       # 0x0
+; CHECK:  .long 0                       # 0x0
+; CHECK:  .long 0                       # 0x0
+; CHECK:  .long 0                       # 0x0
+; CHECK:  .long 8064                    # 0x1f80
+
+; CHECK-LABEL: func_03:
+
+; X86-NOSSE:      fldenv   .LCPI
+; X86-NOSSE-NEXT: retl
+
+; X86-SSE:        fldenv   .LCPI
+; X86-SSE-NEXT:   ldmxcsr .LCPI{{.*}}+28
+; X86-SSE-NEXT:   retl
+
+; X64:            fldenv  .LCPI{{.*}}
+; X64-NEXT:       ldmxcsr .LCPI{{.*}}+28
+; X64-NEXT:       retq
+
+
+declare void @llvm.get.fpenv(i8* %fpenv)
+declare void @llvm.set.fpenv(i8* %fpenv)
+declare void @llvm.reset.fpenv()