[GlobalISel] Implement lowering for G_ISNAN + use it in AArch64

[paquette]

Summary

GlobalISel equivalent to TargetLowering::expandISNAN.

Use it in AArch64 and add a testcase.

[simon_tatham]

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

7378

Sorry to be late spotting this, but I think this fptosi can't be right.

Surely what we need here is the integer that represents the encoding of the FP number, with sign, exponent and mantissa fields?

But this is computing the integer that represents its numerical value, as if from a source-level float→int conversion.

In the AArch64 output code I'm seeing, this fptosi is coming out as an FCVTZS w8,s0 instruction, and with a sensible single-precision NaN in s0 (encoding 0x7fc00000), that instruction is generating zero in w8, which leads to isnan(NaN) returning false.

I don't speak good MIR, but surely something like a bitcast would be the right operation.

[paquette]

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

7378

Maybe it could be just done with a copy? We can't use G_BITCAST because GlobalISel doesn't distinguish between floating point and integer types.

[paquette]

94d3ff09cfa8d7aecf480e54da9a5334e262e76b removes the G_FPTOSI.

[simon_tatham]

Yes, that seems to have fixed it for me – thanks for the quick response!

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

public:
   LegalizeResult lowerBswap(MachineInstr &MI);
   LegalizeResult lowerBitreverse(MachineInstr &MI);
   LegalizeResult lowerReadWriteRegister(MachineInstr &MI);
   LegalizeResult lowerSMULH_UMULH(MachineInstr &MI);
   LegalizeResult lowerSelect(MachineInstr &MI);
   LegalizeResult lowerDIVREM(MachineInstr &MI);
   LegalizeResult lowerAbsToAddXor(MachineInstr &MI);
   LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI);
+  LegalizeResult lowerIsNaN(MachineInstr &MI);
 };
 
 /// Helper function that creates a libcall to the given \p Name using the given
 /// calling convention \p CC.
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
               const CallLowering::ArgInfo &Result,
               ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC);

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

 
 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/Debug.h"
LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
   case G_UDIVREM:
     return lowerDIVREM(MI);
   case G_FSHL:
   case G_FSHR:
     return lowerFunnelShift(MI);
   case G_ROTL:
   case G_ROTR:
     return lowerRotate(MI);
+  case G_ISNAN:
+    return lowerIsNaN(MI);
   }
 }
 
 Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
                                                   Align MinAlign) const {
   // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
   // datalayout for the preferred alignment. Also there should be a target hook
   // for this to allow targets to reduce the alignment and ignore the
LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) {
   Register SrcReg = MI.getOperand(1).getReg();
   LLT Ty = MRI.getType(SrcReg);
   auto Zero = MIRBuilder.buildConstant(Ty, 0).getReg(0);
   auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg).getReg(0);
   MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub);
   MI.eraseFromParent();
   return Legalized;
 }
+
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerIsNaN(MachineInstr &MI) {
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src = MI.getOperand(1).getReg();
+  LLT SrcTy = MRI.getType(Src);
+  if (MI.getFlags() & MachineInstr::NoFPExcept) {
+    // Lower to an unordered comparison.
+    auto Zero = MIRBuilder.buildFConstant(SrcTy, 0.0);
+    MIRBuilder.buildFCmp(CmpInst::Predicate::FCMP_UNO, Dst, Src, Zero);
+    MI.eraseFromParent();
+    return Legalized;
+  }
+
+  // Use integer operations to avoid traps if the argument is SNaN.
+
+  // NaN has all exp bits set and a non zero significand. Therefore:
+  // isnan(V) == exp mask < abs(V)
+  auto FPToSI = MIRBuilder.buildFPTOSI(SrcTy, Src);
+  auto Mask = APInt::getSignedMaxValue(SrcTy.getScalarSizeInBits());
+  auto MaskCst = MIRBuilder.buildConstant(SrcTy, Mask);
+  auto AbsV = MIRBuilder.buildAnd(SrcTy, FPToSI, MaskCst);
+  auto *FloatTy = getFloatTypeForLLT(MI.getMF()->getFunction().getContext(),
+                                     SrcTy.getScalarType());
+  if (!FloatTy)
+    return UnableToLegalize;
+  auto ExpMask = APFloat::getInf(FloatTy->getFltSemantics()).bitcastToAPInt();
+  auto ExpMaskCst = MIRBuilder.buildConstant(SrcTy, ExpMask);
+  MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_SLT, Dst, ExpMaskCst, AbsV);
+  MI.eraseFromParent();
+  return Legalized;
+}

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_CTPOP)
                   {v2s32, v2s32},
                   {v4s32, v4s32},
                   {v4s16, v4s16},
                   {v8s16, v8s16}});
 
   // TODO: Vector types.
   getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT}).lowerIf(isScalar(0));
 
+  getActionDefinitionsBuilder(G_ISNAN).lower();
+
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }
 
 bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
                                           MachineInstr &MI) const {
   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
   MachineRegisterInfo &MRI = *MIRBuilder.getMRI();

llvm/test/CodeGen/AArch64/GlobalISel/legalize-isnan.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=legalizer -global-isel-abort=0 -verify-machineinstrs %s -o - | FileCheck %s
+
+...
+---
+name:            scalar_nofpexcept
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $h0
+
+    ; CHECK-LABEL: name: scalar_nofpexcept
+    ; CHECK: liveins: $h0
+    ; CHECK: %val:_(s16) = COPY $h0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[C]](s32)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %val(s16)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s32) = G_FCMP floatpred(uno), [[FPEXT]](s32), [[FPEXT1]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[FCMP]](s32)
+    ; CHECK: %ext:_(s32) = G_AND [[COPY]], [[C1]]
+    ; CHECK: $w0 = COPY %ext(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %val:_(s16) = COPY $h0
+    %isnan:_(s1) = nofpexcept G_ISNAN %val(s16)
+    %ext:_(s32) = G_ZEXT %isnan(s1)
+    $w0 = COPY %ext(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            vector_nofpexcept
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: vector_nofpexcept
+    ; CHECK: liveins: $d0
+    ; CHECK: %val:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[C]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC]](s16), [[FPTRUNC]](s16), [[FPTRUNC]](s16)
+    ; CHECK: [[FCMP:%[0-9]+]]:_(<4 x s16>) = G_FCMP floatpred(uno), %val(<4 x s16>), [[BUILD_VECTOR]]
+    ; CHECK: %ext:_(<4 x s16>) = COPY [[FCMP]](<4 x s16>)
+    ; CHECK: $d0 = COPY %ext(<4 x s16>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %val:_(<4 x s16>) = COPY $d0
+    %isnan:_(<4 x s1>) = nofpexcept G_ISNAN %val(<4 x s16>)
+    %ext:_(<4 x s16>) = G_ANYEXT %isnan(<4 x s1>)
+    $d0 = COPY %ext(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            scalar_no_flags
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $h0
+
+    ; CHECK-LABEL: name: scalar_no_flags
+    ; CHECK: liveins: $h0
+    ; CHECK: %val:_(s16) = COPY $h0
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %val(s16)
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FPEXT]](s32)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31744
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 16
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[C1]](s32), [[SEXT_INREG]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; CHECK: %ext:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: $w0 = COPY %ext(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %val:_(s16) = COPY $h0
+    %isnan:_(s1) = G_ISNAN %val(s16)
+    %ext:_(s32) = G_ZEXT %isnan(s1)
+    $w0 = COPY %ext(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            vector_no_flags
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: vector_no_flags
+    ; CHECK: liveins: $d0
+    ; CHECK: %val:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(<4 x s16>) = G_FPTOSI %val(<4 x s16>)
+    ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16)
+    ; CHECK: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[FPTOSI]], [[BUILD_VECTOR]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 31744
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(<4 x s16>) = G_ICMP intpred(slt), [[BUILD_VECTOR1]](<4 x s16>), [[AND]]
+    ; CHECK: %isnan:_(<4 x s1>) = G_TRUNC [[ICMP]](<4 x s16>)
+    ; CHECK: %ext:_(<4 x s16>) = G_ANYEXT %isnan(<4 x s1>)
+    ; CHECK: $d0 = COPY %ext(<4 x s16>)
+    ; CHECK: RET_ReallyLR implicit $d0
+    %val:_(<4 x s16>) = COPY $d0
+    %isnan:_(<4 x s1>) = G_ISNAN %val(<4 x s16>)
+    %ext:_(<4 x s16>) = G_ANYEXT %isnan(<4 x s1>)
+    $d0 = COPY %ext(<4 x s16>)
+    RET_ReallyLR implicit $d0