[FPEnv][AArch64] Implement lowering of llvm.set.rounding

[simon_tatham]

llvm/test/CodeGen/AArch64/fpenv.ll

13

This may be outside the scope of your patch, but do you have any idea why this hasn't come out as BIC x8, x8, #0xC00000, but instead, has carefully materialized ~0xC00000 in a register so that it can use AND instead of BIC?

[sepavloff]

llvm/test/CodeGen/AArch64/fpenv.ll

13

Looking at A64 instruction documentation, I see that BIC has only register, register variant.

[simon_tatham]

llvm/test/CodeGen/AArch64/fpenv.ll

13

Yes, sorry – I misremembered what the official name of the instruction was. (I'm more of an AArch32 expert than AArch64.)

But there is an immediate instruction that does what you want. This C source

unsigned long foo(unsigned long x) { return x & ~0xC00000L; }

compiles to this instruction:

and     x0, x0, #0xffffffffff3fffff

[david-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

3493

Should RMMask be uint64_t here since you're creating a MVT::i64 constant?

[sepavloff]

Changed variable type from unsigned to int

[sepavloff]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

3493

Yes, it caused inefficient code generation, as noted by @simon_tatham. How it has type int because it is converted to 64-bit value using sign extension, so produces convenient constant.

llvm/test/CodeGen/AArch64/fpenv.ll

13

This is result of using unsigned for RMMask. When it is converted to 64-bit value, upper 32 bits were zero, this constant cannot be represented as a logical constant in AND instruction, so more complex construct was used for this constant. Using int as the variable type results in sign-extended value, in which upper half is all ones and it can be encoded as immediate in AND.

The generated code becomes better. Thank you!

[simon_tatham]

Looks nicer now, thanks!

llvm/include/llvm/IR/IntrinsicsAArch64.td

 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 
 let TargetPrefix = "aarch64" in {
   class FPCR_Get_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
+  class FPCR_Set_Intrinsic
+    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>;
 }
 
 // FPCR
 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
+def int_aarch64_set_fpcr : FPCR_Set_Intrinsic;
 
 let TargetPrefix = "aarch64" in {
   class Crypto_AES_DataKey_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 
   class Crypto_AES_Data_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 

llvm/lib/Target/AArch64/AArch64ISelLowering.h

static inline bool isDef32(const SDNode &N) {
   unsigned Opc = N.getOpcode();
   return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
          Opc != ISD::CopyFromReg && Opc != ISD::AssertSext &&
          Opc != ISD::AssertZext;
 }
 
 } // end anonymous namespace
 
+namespace AArch64 {
+/// Possible values of current rounding mode, which is specified in bits
+/// 23:22 of FPCR.
+enum Rounding {
+  RN = 0,    // Round to Nearest
+  RP = 1,    // Round towards Plus infinity
+  RM = 2,    // Round towards Minus infinity
+  RZ = 3,    // Round towards Zero
+  rmMask = 3 // Bit mask selecting rounding mode
+};
+
+// Bit position of rounding mode bits in FPCR.
+const unsigned RoundingBitsPos = 22;
+} // namespace AArch64
+
 class AArch64Subtarget;
 class AArch64TargetMachine;
 
 class AArch64TargetLowering : public TargetLowering {
 public:
   explicit AArch64TargetLowering(const TargetMachine &TM,
                                  const AArch64Subtarget &STI);
 
private:
   SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG, unsigned NewOp,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasFullFP16()) {
     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
   }
 
   setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
   setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
+  setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
 
   setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
 
   // Generate outline atomics library calls only if LSE was not specified for
SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
                                   DAG.getConstant(1U << 22, dl, MVT::i32));
   SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
                               DAG.getConstant(22, dl, MVT::i32));
   SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
                             DAG.getConstant(3, dl, MVT::i32));
   return DAG.getMergeValues({AND, Chain}, dl);
 }
 
+SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  SDValue Chain = Op->getOperand(0);
+  SDValue RMValue = Op->getOperand(1);
+
+  // The rounding mode is in bits 23:22 of the FPCR.
+  // The llvm.set.rounding argument value to the rounding mode in FPCR mapping
+  // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
+  // ((arg - 1) & 3) << 22).
+  //
+  // The argument of llvm.set.rounding must be within the segment [0, 3], so
+  // NearestTiesToAway (4) is not handled here. It is responsibility of the code
+  // generated llvm.set.rounding to ensure this condition.
+
+  // Calculate new value of FPCR[23:22].
+  RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
+                        DAG.getConstant(1, DL, MVT::i32));
+  RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
+                        DAG.getConstant(0x3, DL, MVT::i32));
+  RMValue =
+      DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
+                  DAG.getConstant(AArch64::RoundingBitsPos, DL, MVT::i32));
+  RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, RMValue);
+
+  // Get current value of FPCR.
+  SDValue Ops[] = {
+      Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)};
+  SDValue FPCR =
+      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops);
+  Chain = FPCR.getValue(1);
+  FPCR = FPCR.getValue(0);
+
+  // Put new rounding mode into FPSCR[23:22].
+  const int RMMask = ~(AArch64::Rounding::rmMask << AArch64::RoundingBitsPos);
+  FPCR = DAG.getNode(ISD::AND, DL, MVT::i64, FPCR,
+                     DAG.getConstant(RMMask, DL, MVT::i64));
+  FPCR = DAG.getNode(ISD::OR, DL, MVT::i64, FPCR, RMValue);
+  SDValue Ops2[] = {
+      Chain, DAG.getTargetConstant(Intrinsic::aarch64_set_fpcr, DL, MVT::i64),
+      FPCR};
+  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
+}
+
 SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
 
   // If SVE is available then i64 vector multiplications can also be made legal.
   bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64;
 
   if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED, OverrideNEON);
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::FP_TO_UINT:
   case ISD::STRICT_FP_TO_SINT:
   case ISD::STRICT_FP_TO_UINT:
     return LowerFP_TO_INT(Op, DAG);
   case ISD::FSINCOS:
     return LowerFSINCOS(Op, DAG);
   case ISD::FLT_ROUNDS_:
     return LowerFLT_ROUNDS_(Op, DAG);
+  case ISD::SET_ROUNDING:
+    return LowerSET_ROUNDING(Op, DAG);
   case ISD::MUL:
     return LowerMUL(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN:
     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::STORE:
     return LowerSTORE(Op, DAG);
   case ISD::MGATHER:
     return LowerMGATHER(Op, DAG);

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 }
 
 // The cycle counter PMC register is PMCCNTR_EL0.
 let Predicates = [HasPerfMon] in
 def : Pat<(readcyclecounter), (MRS 0xdce8)>;
 
 // FPCR register
 def : Pat<(i64 (int_aarch64_get_fpcr)), (MRS 0xda20)>;
+def : Pat<(int_aarch64_set_fpcr i64:$val), (MSR 0xda20, GPR64:$val)>;
 
 // Generic system instructions
 def SYSxt  : SystemXtI<0, "sys">;
 def SYSLxt : SystemLXtI<1, "sysl">;
 
 def : InstAlias<"sys $op1, $Cn, $Cm, $op2",
                 (SYSxt imm0_7:$op1, sys_cr_op:$Cn,
                  sys_cr_op:$Cm, imm0_7:$op2, XZR)>;

llvm/test/CodeGen/AArch64/fpenv.ll

+; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s
+
+define void @func_02(i32 %rm) {
+  call void @llvm.set.rounding(i32 %rm)
+  ret void
+}
+
+; CHECK-LABEL: func_02:
+; CHECK:       sub   w9, w0, #1
+; CHECK:       mrs   x8, FPCR
+; CHECK:       and   w9, w9, #0x3
+; CHECK:       and   x8, x8, #0xffffffffff3fffff
+; CHECK:       lsl   w9, w9, #22
+; CHECK:       orr   x8, x8, x9
+; CHECK:       msr   FPCR, x8
+; CHECK:       ret
+
+
+define void @func_03() {
+  call void @llvm.set.rounding(i32 0)
+  ret void
+}
+
+; CHECK-LABEL: func_03
+; CHECK:       mrs   x8, FPCR
+; CHECK:       orr   x8, x8, #0xc00000
+; CHECK:       msr   FPCR, x8
+; CHECK:       ret
+
+
+define void @func_04() {
+  call void @llvm.set.rounding(i32 1)
+  ret void
+}
+
+; CHECK-LABEL: func_04
+; CHECK:       mrs   x8, FPCR
+; CHECK:       and   x8, x8, #0xffffffffff3fffff
+; CHECK:       msr   FPCR, x8
+; CHECK:       ret
+
+
+define void @func_05() {
+  call void @llvm.set.rounding(i32 2)
+  ret void
+}
+
+
+; CHECK-LABEL: func_05
+; CHECK:       mrs   x8, FPCR
+; CHECK:       and   x8, x8, #0xffffffffff3fffff
+; CHECK:       orr   x8, x8, #0x400000
+; CHECK:       msr   FPCR, x8
+; CHECK:       ret
+
+
+define void @func_06() {
+  call void @llvm.set.rounding(i32 3)
+  ret void
+}
+
+; CHECK-LABEL: func_06
+; CHECK:       mrs   x8, FPCR
+; CHECK:       and   x8, x8, #0xffffffffff3fffff
+; CHECK:       orr   x8, x8, #0x800000
+; CHECK:       msr   FPCR, x8
+; CHECK:       ret
+
+
+declare void @llvm.set.rounding(i32)