AMDGPU: Improve i16/v2i16 bswap

[foad]

Looks great, thanks!

[arsenm]

8c2c0b3637e7466009d5ab1280b42cbb66844525

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

if (Subtarget->hasFlatAddressSpace()) {
     setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
     setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
   }
 
   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
 
   // FIXME: This should be narrowed to i32, but that only happens if i64 is
   // illegal.
+  // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
   setOperationAction(ISD::BSWAP, MVT::i64, Legal);
   setOperationAction(ISD::BSWAP, MVT::i32, Legal);
 
   // On SI this is s_memtime and s_memrealtime on VI.
   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
   setOperationAction(ISD::TRAP, MVT::Other, Custom);
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom);
 
if (Subtarget->has16BitInsts()) {
     setOperationAction(ISD::ROTR, MVT::i16, Promote);
     setOperationAction(ISD::ROTL, MVT::i16, Promote);
 
     setOperationAction(ISD::SDIV, MVT::i16, Promote);
     setOperationAction(ISD::UDIV, MVT::i16, Promote);
     setOperationAction(ISD::SREM, MVT::i16, Promote);
     setOperationAction(ISD::UREM, MVT::i16, Promote);
 
-    setOperationAction(ISD::BSWAP, MVT::i16, Promote);
     setOperationAction(ISD::BITREVERSE, MVT::i16, Promote);
 
     setOperationAction(ISD::CTTZ, MVT::i16, Promote);
     setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote);
     setOperationAction(ISD::CTLZ, MVT::i16, Promote);
     setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote);
     setOperationAction(ISD::CTPOP, MVT::i16, Promote);
 
for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) {
           break;
         default:
           setOperationAction(Op, VT, Expand);
           break;
         }
       }
     }
 
+    // v_perm_b32 can handle either of these.
+    setOperationAction(ISD::BSWAP, MVT::i16, Legal);
+    setOperationAction(ISD::BSWAP, MVT::v2i16, Legal);
+    setOperationAction(ISD::BSWAP, MVT::v4i16, Custom);
+
     // XXX - Do these do anything? Vector constants turn into build_vector.
     setOperationAction(ISD::Constant, MVT::v2i16, Legal);
     setOperationAction(ISD::ConstantFP, MVT::v2f16, Legal);
 
     setOperationAction(ISD::UNDEF, MVT::v2i16, Legal);
     setOperationAction(ISD::UNDEF, MVT::v2f16, Legal);
 
     setOperationAction(ISD::STORE, MVT::v2i16, Promote);

 //===----------------------------------------------------------------------===//
 
 // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
 // wider vector type is legal.
 SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
                                              SelectionDAG &DAG) const {
   unsigned Opc = Op.getOpcode();
   EVT VT = Op.getValueType();
-  assert(VT == MVT::v4f16);
+  assert(VT == MVT::v4f16 || VT == MVT::v4i16);
 
   SDValue Lo, Hi;
   std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
 
   SDLoc SL(Op);
   SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
                              Op->getFlags());
   SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
case ISD::FP_ROUND:
     return lowerFP_ROUND(Op, DAG);
   case ISD::TRAP:
     return lowerTRAP(Op, DAG);
   case ISD::DEBUGTRAP:
     return lowerDEBUGTRAP(Op, DAG);
   case ISD::FABS:
   case ISD::FNEG:
   case ISD::FCANONICALIZE:
+  case ISD::BSWAP:
     return splitUnaryVectorOp(Op, DAG);
   case ISD::FMINNUM:
   case ISD::FMAXNUM:
     return lowerFMINNUM_FMAXNUM(Op, DAG);
   case ISD::FMA:
     return splitTernaryVectorOp(Op, DAG);
   case ISD::SHL:
   case ISD::SRA:

llvm/lib/Target/AMDGPU/SIInstructions.td

def : GCNPat <
   (V_PERM_B32 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub1),
               (S_MOV_B32 (i32 0x00010203))),
   sub0,
   (V_PERM_B32 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub0),
               (S_MOV_B32 (i32 0x00010203))),
   sub1)
 >;
 
+// Magic number: 1 | (0 << 8) | (12 << 16) | (12 << 24)
+// The 12s emit 0s.
+def : GCNPat <
+  (i16 (bswap i16:$a)),
+  (V_PERM_B32 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001)))
+>;
+
+def : GCNPat <
+  (i32 (zext (bswap i16:$a))),
+  (V_PERM_B32 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001)))
+>;
+
+// Magic number: 1 | (0 << 8) | (3 << 16) | (2 << 24)
+def : GCNPat <
+  (v2i16 (bswap v2i16:$a)),
+  (V_PERM_B32 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x02030001)))
+>;
+
 }
 
 let OtherPredicates = [NoFP16Denormals] in {
 def : GCNPat<
   (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),
   (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src, 0, 0)
 >;
 

llvm/test/CodeGen/AMDGPU/bswap.ll

 ; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: missing_truncate_promote_bswap:
 ; VI:       ; %bb.0: ; %bb
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s4, 0x10203
+; VI-NEXT:    s_mov_b32 s4, 0xc0c0001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %tmp = trunc i32 %arg to i16
   %tmp1 = call i16 @llvm.bswap.i16(i16 %tmp)
   %tmp2 = bitcast i16 %tmp1 to half
   %tmp3 = fpext half %tmp2 to float
   ret float %tmp3
 }
 
 define i16 @v_bswap_i16(i16 %src) {
 ; SI-LABEL: v_bswap_i16:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v0, v0, 8
 ; SI-NEXT:    v_alignbit_b32 v0, v0, v0, 24
 ; SI-NEXT:    s_mov_b32 s4, 0xff00ff
 ; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s4, 0x10203
+; VI-NEXT:    s_mov_b32 s4, 0xc0c0001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %bswap = call i16 @llvm.bswap.i16(i16 %src)
   ret i16 %bswap
 }
 
 define i32 @v_bswap_i16_zext_to_i32(i16 %src) {
 ; SI-LABEL: v_bswap_i16_zext_to_i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v0, v0, 8
 ; SI-NEXT:    v_alignbit_b32 v0, v0, v0, 24
 ; SI-NEXT:    s_mov_b32 s4, 0xff00ff
 ; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_i16_zext_to_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s4, 0x10203
+; VI-NEXT:    s_mov_b32 s4, 0xc0c0001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %bswap = call i16 @llvm.bswap.i16(i16 %src)
   %zext = zext i16 %bswap to i32
   ret i32 %zext
 }
 
 define i32 @v_bswap_i16_sext_to_i32(i16 %src) {
 ; SI-LABEL: v_bswap_i16_sext_to_i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    v_alignbit_b32 v1, v0, v0, 8
 ; SI-NEXT:    v_alignbit_b32 v0, v0, v0, 24
 ; SI-NEXT:    s_mov_b32 s4, 0xff00ff
 ; SI-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; SI-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_i16_sext_to_i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s4, 0x10203
+; VI-NEXT:    s_mov_b32 s4, 0xc0c0001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; VI-NEXT:    v_bfe_i32 v0, v0, 0, 16
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %bswap = call i16 @llvm.bswap.i16(i16 %src)
   %zext = sext i16 %bswap to i32
   ret i32 %zext
 }
 
 define <2 x i16> @v_bswap_v2i16(<2 x i16> %src) {
 ; SI-LABEL: v_bswap_v2i16:

 ; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_v2i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; VI-NEXT:    s_mov_b32 s4, 0x10203
-; VI-NEXT:    v_perm_b32 v1, 0, v1, s4
+; VI-NEXT:    s_mov_b32 s4, 0x2030001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %bswap = call <2 x i16> @llvm.bswap.v2i16(<2 x i16> %src)
   ret <2 x i16> %bswap
 }
 
 define <3 x i16> @v_bswap_v3i16(<3 x i16> %src) {
 ; SI-LABEL: v_bswap_v3i16:
 ; SI:       ; %bb.0:

 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
 ; SI-NEXT:    v_alignbit_b32 v1, v2, v0, 16
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_v3i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    s_mov_b32 s4, 0x10203
-; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; VI-NEXT:    v_perm_b32 v1, 0, v1, s4
-; VI-NEXT:    v_perm_b32 v2, 0, v2, s4
+; VI-NEXT:    s_mov_b32 s4, 0x2030001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_perm_b32 v1, 0, v1, s4
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %bswap = call <3 x i16> @llvm.bswap.v3i16(<3 x i16> %src)
   ret <3 x i16> %bswap
 }
 
 define <4 x i16> @v_bswap_v4i16(<4 x i16> %src) {
 ; SI-LABEL: v_bswap_v4i16:
 ; SI:       ; %bb.0:

 ; SI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; SI-NEXT:    v_alignbit_b32 v1, v2, v0, 16
 ; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: v_bswap_v4i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; VI-NEXT:    s_mov_b32 s4, 0x10203
-; VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; VI-NEXT:    v_perm_b32 v2, 0, v2, s4
-; VI-NEXT:    v_perm_b32 v1, 0, v1, s4
-; VI-NEXT:    v_perm_b32 v3, 0, v3, s4
+; VI-NEXT:    s_mov_b32 s4, 0x2030001
 ; VI-NEXT:    v_perm_b32 v0, 0, v0, s4
-; VI-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT:    v_perm_b32 v1, 0, v1, s4
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %bswap = call <4 x i16> @llvm.bswap.v4i16(<4 x i16> %src)
   ret <4 x i16> %bswap
 }