[RISCV] Custom type legalize i32 fshl/fshr on RV64 with Zbt.

[craig.topper]

Summary

This adds custom opcodes for FSLW/FSRW so we can type legalize
fshl/fshr without needing to match a sign_extend_inreg.

I've used the operand order from fshl/fshr to make the isel
pattern similar to the non-W form. It was also hard to decide
another order since the register instruction has the shift amount
as the second operand, but the immediate instruction has it as
the third operand.

[craig.topper]

Rebase

[frasercrmck]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1100

I think it's worth introducing a scope here, so someone adding a case after this won't have to.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1100

Good catch. Thanks!

[craig.topper]

Address review comment

[frasercrmck]

LGTM.

llvm/lib/Target/RISCV/RISCVISelLowering.h

enum NodeType : unsigned {
   // at instruction selection time.
   DIVW,
   DIVUW,
   REMUW,
   // RV64IB rotates, directly matching the semantics of the named RISC-V
   // instructions.
   ROLW,
   RORW,
+  // RV64IB funnel shifts, with the semantics of the named RISC-V instructions,
+  // but the same operand order as fshl/fshr intrinsics.
+  FSRW,
+  FSLW,
   // FPR32<->GPR transfer operations for RV64. Needed as an i32<->f32 bitcast
   // is not legal on RV64. FMV_W_X_RV64 matches the semantics of the FMV.W.X.
   // FMV_X_ANYEXTW_RV64 is similar to FMV.X.W but has an any-extended result.
   // This is a more convenient semantic for producing dagcombines that remove
   // unnecessary GPR->FPR->GPR moves.
   FMV_W_X_RV64,
   FMV_X_ANYEXTW_RV64,
   // READ_CYCLE_WIDE - A read of the 64-bit cycle CSR on a 32-bit target

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

if (!Subtarget.hasStdExtZbb()) {
     setOperationAction(ISD::CTTZ, XLenVT, Expand);
     setOperationAction(ISD::CTLZ, XLenVT, Expand);
     setOperationAction(ISD::CTPOP, XLenVT, Expand);
   }
 
   if (Subtarget.hasStdExtZbt()) {
     setOperationAction(ISD::FSHL, XLenVT, Legal);
     setOperationAction(ISD::FSHR, XLenVT, Legal);
+
+    if (Subtarget.is64Bit()) {
+      setOperationAction(ISD::FSHL, MVT::i32, Custom);
+      setOperationAction(ISD::FSHR, MVT::i32, Custom);
+    }
   }
 
   ISD::CondCode FPCCToExtend[] = {
       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
       ISD::SETGE,  ISD::SETNE};
 
   ISD::NodeType FPOpToExtend[] = {
case ISD::BITREVERSE: {
     SDValue GREVIW = DAG.getNode(RISCVISD::GREVIW, DL, MVT::i64, NewOp0,
                                  DAG.getTargetConstant(Imm, DL,
                                                        Subtarget.getXLenVT()));
     // ReplaceNodeResults requires we maintain the same type for the return
     // value.
     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, GREVIW));
     break;
   }
+  case ISD::FSHL:
+  case ISD::FSHR: {
+    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+           Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
+    SDValue NewOp0 =
+        DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
+    SDValue NewOp1 =
+        DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
+    SDValue NewOp2 =
+        DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
+    // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
+    // Mask the shift amount to 5 bits.
+    NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
+                         DAG.getConstant(0x1f, DL, MVT::i64));
+    unsigned Opc =
+        N->getOpcode() == ISD::FSHL ? RISCVISD::FSLW : RISCVISD::FSRW;
+    SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewOp2);
+    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
+    break;
+  }
   }
 }
 
 // A structure to hold one of the bit-manipulation patterns below. Together, a
 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
 //   (or (and (shl x, 1), 0xAAAAAAAA),
 //       (and (srl x, 1), 0x55555555))
 struct RISCVBitmanipPat {
case RISCVISD::RORW: {
     if (SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI) ||
         SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)) {
       if (N->getOpcode() != ISD::DELETED_NODE)
         DCI.AddToWorklist(N);
       return SDValue(N, 0);
     }
     break;
   }
+  case RISCVISD::FSLW:
+  case RISCVISD::FSRW: {
+    // Only the lower 32 bits of Values and lower 6 bits of shift amount are
+    // read.
+    SDValue Op0 = N->getOperand(0);
+    SDValue Op1 = N->getOperand(1);
+    SDValue ShAmt = N->getOperand(2);
+    APInt OpMask = APInt::getLowBitsSet(Op0.getValueSizeInBits(), 32);
+    APInt ShAmtMask = APInt::getLowBitsSet(ShAmt.getValueSizeInBits(), 6);
+    if (SimplifyDemandedBits(Op0, OpMask, DCI) ||
+        SimplifyDemandedBits(Op1, OpMask, DCI) ||
+        SimplifyDemandedBits(ShAmt, ShAmtMask, DCI)) {
+      if (N->getOpcode() != ISD::DELETED_NODE)
+        DCI.AddToWorklist(N);
+      return SDValue(N, 0);
+    }
+    break;
+  }
   case RISCVISD::GREVIW:
   case RISCVISD::GORCIW: {
     // Only the lower 32 bits of the first operand are read
     SDValue Op0 = N->getOperand(0);
     APInt Mask = APInt::getLowBitsSet(Op0.getValueSizeInBits(), 32);
     if (SimplifyDemandedBits(Op0, Mask, DCI)) {
       if (N->getOpcode() != ISD::DELETED_NODE)
         DCI.AddToWorklist(N);
unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
   case RISCVISD::SRLW:
   case RISCVISD::DIVW:
   case RISCVISD::DIVUW:
   case RISCVISD::REMUW:
   case RISCVISD::ROLW:
   case RISCVISD::RORW:
   case RISCVISD::GREVIW:
   case RISCVISD::GORCIW:
+  case RISCVISD::FSLW:
+  case RISCVISD::FSRW:
     // TODO: As the result is sign-extended, this is conservatively correct. A
     // more precise answer could be calculated for SRAW depending on known
     // bits in the shift amount.
     return 33;
   }
 
   return 1;
 }
#define NODE_NAME_CASE(NODE)                                                   \
   NODE_NAME_CASE(SLLW)
   NODE_NAME_CASE(SRAW)
   NODE_NAME_CASE(SRLW)
   NODE_NAME_CASE(DIVW)
   NODE_NAME_CASE(DIVUW)
   NODE_NAME_CASE(REMUW)
   NODE_NAME_CASE(ROLW)
   NODE_NAME_CASE(RORW)
+  NODE_NAME_CASE(FSLW)
+  NODE_NAME_CASE(FSRW)
   NODE_NAME_CASE(FMV_W_X_RV64)
   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
   NODE_NAME_CASE(READ_CYCLE_WIDE)
   NODE_NAME_CASE(GREVI)
   NODE_NAME_CASE(GREVIW)
   NODE_NAME_CASE(GORCI)
   NODE_NAME_CASE(GORCIW)
   }

llvm/lib/Target/RISCV/RISCVInstrInfoB.td

 // ratified yet.
 //
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
 //===----------------------------------------------------------------------===//
 
-def riscv_rolw      : SDNode<"RISCVISD::ROLW", SDTIntShiftOp>;
-def riscv_rorw      : SDNode<"RISCVISD::RORW", SDTIntShiftOp>;
+def riscv_rolw : SDNode<"RISCVISD::ROLW", SDTIntShiftOp>;
+def riscv_rorw : SDNode<"RISCVISD::RORW", SDTIntShiftOp>;
+def riscv_fslw : SDNode<"RISCVISD::FSLW", SDTIntShiftDOp>;
+def riscv_fsrw : SDNode<"RISCVISD::FSRW", SDTIntShiftDOp>;
 
 def UImmLog2XLenHalfAsmOperand : AsmOperandClass {
   let Name = "UImmLog2XLenHalf";
   let RenderMethod = "addImmOperands";
   let DiagnosticType = "InvalidUImmLog2XLenHalf";
 }
 
 def shfl_uimm : Operand<XLenVT>, ImmLeaf<XLenVT, [{

 } // Predicates = [HasStdExtZbb, IsRV64]
 
 let Predicates = [HasStdExtZbp, IsRV64] in {
 def : Pat<(riscv_greviw GPR:$rs1, timm:$shamt), (GREVIW GPR:$rs1, timm:$shamt)>;
 def : Pat<(riscv_gorciw GPR:$rs1, timm:$shamt), (GORCIW GPR:$rs1, timm:$shamt)>;
 } // Predicates = [HasStdExtZbp, IsRV64]
 
 let Predicates = [HasStdExtZbt, IsRV64] in {
-def : Pat<(sext_inreg (fshl GPR:$rs1, (shl GPR:$rs3, (i64 32)),
-                            (and GPR:$rs2, (i64 31))),
-                      i32),
-          (FSLW GPR:$rs1, (ANDI GPR:$rs2, 31), GPR:$rs3)>;
-def : Pat<(sext_inreg (fshr GPR:$rs3, (shl GPR:$rs1, (i64 32)),
-                            (or GPR:$rs2, (i64 32))),
-                      i32),
-          (FSRW GPR:$rs1, (ANDI GPR:$rs2, 31), GPR:$rs3)>;
-def : Pat<(sext_inreg (fshr GPR:$rs3, (shl GPR:$rs1, (i64 32)),
-                            uimm6gt32:$shamt),
-                      i32),
-          (FSRIW GPR:$rs1, GPR:$rs3, (ImmSub32 uimm6gt32:$shamt))>;
-def : Pat<(sext_inreg (fshl GPR:$rs3, (shl GPR:$rs1, (i64 32)),
-                            uimm5:$shamt),
-                      i32),
+def : Pat<(riscv_fslw GPR:$rs1, GPR:$rs3, GPR:$rs2),
+          (FSLW GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+def : Pat<(riscv_fsrw GPR:$rs3, GPR:$rs1, GPR:$rs2),
+          (FSRW GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
+def : Pat<(riscv_fsrw GPR:$rs3, GPR:$rs1, uimm5:$shamt),
+          (FSRIW GPR:$rs1, GPR:$rs3, uimm5:$shamt)>;
+def : Pat<(riscv_fslw GPR:$rs3, GPR:$rs1, uimm5:$shamt),
           (FSRIW GPR:$rs1, GPR:$rs3, (ImmROTL2RW uimm5:$shamt))>;
 } // Predicates = [HasStdExtZbt, IsRV64]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
 def : Pat<(add (ctlz (and GPR:$rs1, (i64 0xFFFFFFFF))), (i64 -32)),
           (CLZW GPR:$rs1)>;
 // computeKnownBits can't figure out that the and mask on the add result is
 // unnecessary so we need to pattern match it away.

llvm/test/CodeGen/RISCV/rv64Zbt.ll

 ; RV64IBT-NEXT:    andi a2, a2, 31
 ; RV64IBT-NEXT:    fslw a0, a0, a1, a2
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
   ret i32 %1
 }
 
 ; Similar to fshl_i32 but result is not sign extended.
-; FIXME: This should use fslw
 define void @fshl_i32_nosext(i32 signext %a, i32 signext %b, i32 signext %c, i32* %x) nounwind {
 ; RV64I-LABEL: fshl_i32_nosext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    srli a1, a1, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    andi a1, a2, 31
 ; RV64I-NEXT:    sll a0, a0, a1
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    sw a0, 0(a3)
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: fshl_i32_nosext:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    slli a1, a1, 32
 ; RV64IB-NEXT:    andi a2, a2, 31
-; RV64IB-NEXT:    andi a2, a2, 63
-; RV64IB-NEXT:    fsl a0, a0, a1, a2
+; RV64IB-NEXT:    fslw a0, a0, a1, a2
 ; RV64IB-NEXT:    sw a0, 0(a3)
 ; RV64IB-NEXT:    ret
 ;
 ; RV64IBT-LABEL: fshl_i32_nosext:
 ; RV64IBT:       # %bb.0:
-; RV64IBT-NEXT:    slli a1, a1, 32
 ; RV64IBT-NEXT:    andi a2, a2, 31
-; RV64IBT-NEXT:    andi a2, a2, 63
-; RV64IBT-NEXT:    fsl a0, a0, a1, a2
+; RV64IBT-NEXT:    fslw a0, a0, a1, a2
 ; RV64IBT-NEXT:    sw a0, 0(a3)
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
   store i32 %1, i32* %x
   ret void
 }
 
 declare i64 @llvm.fshl.i64(i64, i64, i64)

 ; RV64IBT-NEXT:    andi a2, a2, 31
 ; RV64IBT-NEXT:    fsrw a0, a1, a0, a2
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
   ret i32 %1
 }
 
 ; Similar to fshr_i32 but result is not sign extended.
-; FIXME: This should use fsrw
 define void @fshr_i32_nosext(i32 signext %a, i32 signext %b, i32 signext %c, i32* %x) nounwind {
 ; RV64I-LABEL: fshr_i32_nosext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    srli a1, a1, 32
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    andi a1, a2, 31
 ; RV64I-NEXT:    srl a0, a0, a1
 ; RV64I-NEXT:    sw a0, 0(a3)
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: fshr_i32_nosext:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    slli a1, a1, 32
-; RV64IB-NEXT:    ori a2, a2, 32
-; RV64IB-NEXT:    andi a2, a2, 63
-; RV64IB-NEXT:    fsr a0, a1, a0, a2
+; RV64IB-NEXT:    andi a2, a2, 31
+; RV64IB-NEXT:    fsrw a0, a1, a0, a2
 ; RV64IB-NEXT:    sw a0, 0(a3)
 ; RV64IB-NEXT:    ret
 ;
 ; RV64IBT-LABEL: fshr_i32_nosext:
 ; RV64IBT:       # %bb.0:
-; RV64IBT-NEXT:    slli a1, a1, 32
-; RV64IBT-NEXT:    ori a2, a2, 32
-; RV64IBT-NEXT:    andi a2, a2, 63
-; RV64IBT-NEXT:    fsr a0, a1, a0, a2
+; RV64IBT-NEXT:    andi a2, a2, 31
+; RV64IBT-NEXT:    fsrw a0, a1, a0, a2
 ; RV64IBT-NEXT:    sw a0, 0(a3)
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
   store i32 %1, i32* %x
   ret void
 }
 
 declare i64 @llvm.fshr.i64(i64, i64, i64)

 ; RV64IBT:       # %bb.0:
 ; RV64IBT-NEXT:    fsriw a0, a1, a0, 5
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 5)
   ret i32 %1
 }
 
 ; Similar to fshr_i32 but result is not sign extended.
-; FIXME: This should use fsriw
 define void @fshri_i32_nosext(i32 signext %a, i32 signext %b, i32* %x) nounwind {
 ; RV64I-LABEL: fshri_i32_nosext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srliw a1, a1, 5
 ; RV64I-NEXT:    slli a0, a0, 27
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    sw a0, 0(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: fshri_i32_nosext:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    slli a1, a1, 32
-; RV64IB-NEXT:    fsri a0, a1, a0, 37
+; RV64IB-NEXT:    fsriw a0, a1, a0, 5
 ; RV64IB-NEXT:    sw a0, 0(a2)
 ; RV64IB-NEXT:    ret
 ;
 ; RV64IBT-LABEL: fshri_i32_nosext:
 ; RV64IBT:       # %bb.0:
-; RV64IBT-NEXT:    slli a1, a1, 32
-; RV64IBT-NEXT:    fsri a0, a1, a0, 37
+; RV64IBT-NEXT:    fsriw a0, a1, a0, 5
 ; RV64IBT-NEXT:    sw a0, 0(a2)
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 5)
   store i32 %1, i32* %x
   ret void
 }
 
 define i64 @fshri_i64(i64 %a, i64 %b) nounwind {

 ; RV64IBT:       # %bb.0:
 ; RV64IBT-NEXT:    fsriw a0, a1, a0, 27
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 5)
   ret i32 %1
 }
 
 ; Similar to fshl_i32 but result is not sign extended.
-; FIXME: This should use fsriw
 define void @fshli_i32_nosext(i32 signext %a, i32 signext %b, i32* %x) nounwind {
 ; RV64I-LABEL: fshli_i32_nosext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    srliw a1, a1, 27
 ; RV64I-NEXT:    slli a0, a0, 5
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    sw a0, 0(a2)
 ; RV64I-NEXT:    ret
 ;
 ; RV64IB-LABEL: fshli_i32_nosext:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    slli a1, a1, 32
-; RV64IB-NEXT:    fsri a0, a1, a0, 59
+; RV64IB-NEXT:    fsriw a0, a1, a0, 27
 ; RV64IB-NEXT:    sw a0, 0(a2)
 ; RV64IB-NEXT:    ret
 ;
 ; RV64IBT-LABEL: fshli_i32_nosext:
 ; RV64IBT:       # %bb.0:
-; RV64IBT-NEXT:    slli a1, a1, 32
-; RV64IBT-NEXT:    fsri a0, a1, a0, 59
+; RV64IBT-NEXT:    fsriw a0, a1, a0, 27
 ; RV64IBT-NEXT:    sw a0, 0(a2)
 ; RV64IBT-NEXT:    ret
   %1 = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 5)
   store i32 %1, i32* %x
   ret void
 }
 
 define i64 @fshli_i64(i64 %a, i64 %b) nounwind {