[AArch64][SME]: Make 'Expand' the default action for all Ops.

[hassnaa-arm]

Summary

By default expand all operations, then change to Custom/Legal if needed.
[AArch64][SME]: By default, Expand all the operations.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1442

Please move this loop where it marks all the operations as Expand into addTypeForStreamingSVE.

1664–1666

Can you add a comment for these operations that we mark them as Legal because we can use streaming-compatible NEON instructions for these?

4329

Can you split out this change, along with marking BITCAST as custom, into a separate patch?

[hassnaa-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

4329

Removing BITCAST from this patch will cause a crash to some testing files, is that okay ?

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

4329

That is not okay :) It just means you need to reorder the patch so the other one comes first.

[hassnaa-arm]

Mark BITCAST as Legal, no need to Custom-lower it.

[hassnaa-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

4329

I found that I was mistaken about custom-lowering BITCAST. No need for making it custom.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1668

Please add newline above.

1674

Please add newline above.

[hassnaa-arm]

Add new empty lines

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->forceStreamingCompatibleSVE()) {
       setTruncStoreAction(MVT::v1f64, MVT::v1f16, Custom);
       setTruncStoreAction(MVT::v2f64, MVT::v2f16, Custom);
       setTruncStoreAction(MVT::v4f64, MVT::v4f16, Custom);
       setTruncStoreAction(MVT::v1f64, MVT::v1f32, Custom);
       setTruncStoreAction(MVT::v2f64, MVT::v2f32, Custom);
       setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom);
       for (MVT VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
                      MVT::v4i32, MVT::v1i64, MVT::v2i64})
         addTypeForStreamingSVE(VT);
 
       for (MVT VT :
            {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v2f64})
         addTypeForStreamingSVE(VT);
     }
 
     // NOTE: Currently this has to happen after computeRegisterProperties rather
     // than the preferred option of combining it with the addRegisterClass call.
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
   // The whilelo instruction only works with i32 or i64 scalar inputs.
   if (OpVT != MVT::i32 && OpVT != MVT::i64)
     return true;
 
   return false;
 }
 
 void AArch64TargetLowering::addTypeForStreamingSVE(MVT VT) {
+  // By default set all operations to Expand,
+  // then change to Legal/Custom if needed.
+  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
+    setOperationAction(Op, VT, Expand);
+  // STORE, LOAD, SCALAR_TO_VECTOR and BITCAST are natively supported,
+  // so no need to Custom/Expand them.
+  setOperationAction(ISD::STORE, VT, Legal);
+  setOperationAction(ISD::LOAD, VT, Legal);
+  setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal);
+  setOperationAction(ISD::BITCAST, VT, Legal);
+  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
+  setOperationAction(ISD::MLOAD, VT, Custom);
+  setOperationAction(ISD::MSTORE, VT, Custom);
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
+  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+  setOperationAction(ISD::SDIV, VT, Custom);
+  setOperationAction(ISD::SHL, VT, Custom);
+  setOperationAction(ISD::SRA, VT, Custom);
+  setOperationAction(ISD::SRL, VT, Custom);
+  setOperationAction(ISD::OR, VT, Custom);
+  setOperationAction(ISD::SETCC, VT, Custom);
+  setOperationAction(ISD::UDIV, VT, Custom);
+  setOperationAction(ISD::SINT_TO_FP, VT, Custom);
+  setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+  setOperationAction(ISD::FP_TO_UINT, VT, Custom);
+  setOperationAction(ISD::UINT_TO_FP, VT, Custom);
+  setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
+  setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
+  setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
+  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+  setOperationAction(ISD::FCOPYSIGN, VT, Custom);
   setOperationAction(ISD::ANY_EXTEND, VT, Custom);
   setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
   setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
   setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
   setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
   setOperationAction(ISD::AND, VT, Custom);
   setOperationAction(ISD::ADD, VT, Custom);
   setOperationAction(ISD::SUB, VT, Custom);

 
 static MVT getSVEContainerType(EVT ContentTy);
 
 SDValue AArch64TargetLowering::LowerBITCAST(SDValue Op,
                                             SelectionDAG &DAG) const {
   EVT OpVT = Op.getValueType();
   EVT ArgVT = Op.getOperand(0).getValueType();
 
   if (useSVEForFixedLengthVectorVT(OpVT))
     return LowerFixedLengthBitcastToSVE(Op, DAG);
 
   if (OpVT.isScalableVector()) {
     // Bitcasting between unpacked vector types of different element counts is
     // not a NOP because the live elements are laid out differently.
     //                01234567
     // e.g. nxv2i32 = XX??XX??
     //      nxv4f16 = X?X?X?X?

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll

 
 define void @select_v16f16(ptr %a, ptr %b, i1 %mask) #0 {
 ; CHECK-LABEL: select_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w2, #0x1
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    csetm w8, ne
 ; CHECK-NEXT:    ldr q1, [x0, #16]
-; CHECK-NEXT:    ldr q2, [x1]
-; CHECK-NEXT:    ldr q3, [x1, #16]
+; CHECK-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEXT:    ldr q3, [x1]
 ; CHECK-NEXT:    mov z4.h, w8
 ; CHECK-NEXT:    bic z2.d, z2.d, z4.d
 ; CHECK-NEXT:    and z0.d, z0.d, z4.d
 ; CHECK-NEXT:    bic z3.d, z3.d, z4.d
 ; CHECK-NEXT:    and z1.d, z1.d, z4.d
-; CHECK-NEXT:    orr z0.d, z0.d, z2.d
-; CHECK-NEXT:    orr z1.d, z1.d, z3.d
+; CHECK-NEXT:    orr z0.d, z0.d, z3.d
+; CHECK-NEXT:    orr z1.d, z1.d, z2.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load volatile <16 x half>, ptr %a
   %op2 = load volatile <16 x half>, ptr %b
   %sel = select i1 %mask, <16 x half> %op1, <16 x half> %op2
   store <16 x half> %sel, ptr %a
   ret void
 }
; CHECK-NEXT:    ret
   %sel = select i1 %mask, <4 x float> %op1, <4 x float> %op2
   ret <4 x float> %sel
 }
 
 define void @select_v8f32(ptr %a, ptr %b, i1 %mask) #0 {
 ; CHECK-LABEL: select_v8f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w2, #0x1
-; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    csetm w8, ne
-; CHECK-NEXT:    ldr q1, [x0, #16]
+; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    mvn w9, w8
-; CHECK-NEXT:    ldr q2, [x1]
-; CHECK-NEXT:    ldr q3, [x1, #16]
+; CHECK-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEXT:    ldr q3, [x1]
 ; CHECK-NEXT:    mov z4.s, w8
 ; CHECK-NEXT:    mov z5.s, w9
-; CHECK-NEXT:    and z1.d, z1.d, z4.d
 ; CHECK-NEXT:    and z0.d, z0.d, z4.d
-; CHECK-NEXT:    and z2.d, z2.d, z5.d
+; CHECK-NEXT:    and z1.d, z1.d, z4.d
 ; CHECK-NEXT:    and z3.d, z3.d, z5.d
-; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    and z2.d, z2.d, z5.d
 ; CHECK-NEXT:    orr z1.d, z1.d, z3.d
-; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load volatile <8 x float>, ptr %a
   %op2 = load volatile <8 x float>, ptr %b
   %sel = select i1 %mask, <8 x float> %op1, <8 x float> %op2
   store <8 x float> %sel, ptr %a
   ret void
 }
 
; CHECK-NEXT:    ret
   %sel = select i1 %mask, <2 x double> %op1, <2 x double> %op2
   ret <2 x double> %sel
 }
 
 define void @select_v4f64(ptr %a, ptr %b, i1 %mask) #0 {
 ; CHECK-LABEL: select_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    tst w2, #0x1
-; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    csetm x8, ne
-; CHECK-NEXT:    ldr q1, [x0, #16]
+; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    mvn x9, x8
-; CHECK-NEXT:    ldr q2, [x1]
-; CHECK-NEXT:    ldr q3, [x1, #16]
+; CHECK-NEXT:    ldr q2, [x1, #16]
+; CHECK-NEXT:    ldr q3, [x1]
 ; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    mov z5.d, x9
-; CHECK-NEXT:    and z1.d, z1.d, z4.d
 ; CHECK-NEXT:    and z0.d, z0.d, z4.d
-; CHECK-NEXT:    and z2.d, z2.d, z5.d
+; CHECK-NEXT:    and z1.d, z1.d, z4.d
 ; CHECK-NEXT:    and z3.d, z3.d, z5.d
-; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    and z2.d, z2.d, z5.d
 ; CHECK-NEXT:    orr z1.d, z1.d, z3.d
-; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load volatile <4 x double>, ptr %a
   %op2 = load volatile <4 x double>, ptr %b
   %sel = select i1 %mask, <4 x double> %op1, <4 x double> %op2
   store <4 x double> %sel, ptr %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll

 ;
 ; FCVTZU D -> D
 ;
 
 define <1 x i64> @fcvtzu_v1f64_v1i64(<1 x double> %op1) #0 {
 ; CHECK-LABEL: fcvtzu_v1f64_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    fcvtzu x8, d0
-; CHECK-NEXT:    mov z0.d, x8
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = fptoui <1 x double> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
 
 define <2 x i64> @fcvtzu_v2f64_v2i64(<2 x double> %op1) #0 {
 ; CHECK-LABEL: fcvtzu_v2f64_v2i64:

 ;
 ; FCVTZS D -> D
 ;
 
 define <1 x i64> @fcvtzs_v1f64_v1i64(<1 x double> %op1) #0 {
 ; CHECK-LABEL: fcvtzs_v1f64_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    mov z0.d, x8
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = fptosi <1 x double> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
 
 define <2 x i64> @fcvtzs_v2f64_v2i64(<2 x double> %op1) #0 {
 ; CHECK-LABEL: fcvtzs_v2f64_v2i64:

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll

 ; CHECK-NEXT:    ret
   %sel = select <8 x i1> %mask, <8 x half> %op1, <8 x half> %op2
   ret <8 x half> %sel
 }
 
 define void @select_v16f16(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: select_v16f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    ldp q3, q2, [x1]
-; CHECK-NEXT:    fcmeq p1.h, p0/z, z1.h, z2.h
-; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    fcmeq p1.h, p0/z, z0.h, z2.h
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z1.h, z3.h
 ; CHECK-NEXT:    mov z4.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z5.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    bic z2.d, z2.d, z4.d
 ; CHECK-NEXT:    bic z3.d, z3.d, z5.d
-; CHECK-NEXT:    and z0.d, z0.d, z5.d
-; CHECK-NEXT:    and z1.d, z1.d, z4.d
-; CHECK-NEXT:    orr z0.d, z0.d, z3.d
-; CHECK-NEXT:    orr z1.d, z1.d, z2.d
-; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    and z1.d, z1.d, z5.d
+; CHECK-NEXT:    and z0.d, z0.d, z4.d
+; CHECK-NEXT:    orr z1.d, z1.d, z3.d
+; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %mask = fcmp oeq <16 x half> %op1, %op2
   %sel = select <16 x i1> %mask, <16 x half> %op1, <16 x half> %op2
   store <16 x half> %sel, ptr %a
   ret void
 }

 ; CHECK-NEXT:    ret
   %sel = select <4 x i1> %mask, <4 x float> %op1, <4 x float> %op2
   ret <4 x float> %sel
 }
 
 define void @select_v8f32(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: select_v8f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
 ; CHECK-NEXT:    ldp q3, q2, [x1]
-; CHECK-NEXT:    fcmeq p1.s, p0/z, z1.s, z2.s
-; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, z2.s
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z1.s, z3.s
 ; CHECK-NEXT:    mov z4.s, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z5.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    bic z2.d, z2.d, z4.d
 ; CHECK-NEXT:    bic z3.d, z3.d, z5.d
-; CHECK-NEXT:    and z0.d, z0.d, z5.d
-; CHECK-NEXT:    and z1.d, z1.d, z4.d
-; CHECK-NEXT:    orr z0.d, z0.d, z3.d
-; CHECK-NEXT:    orr z1.d, z1.d, z2.d
-; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    and z1.d, z1.d, z5.d
+; CHECK-NEXT:    and z0.d, z0.d, z4.d
+; CHECK-NEXT:    orr z1.d, z1.d, z3.d
+; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %mask = fcmp oeq <8 x float> %op1, %op2
   %sel = select <8 x i1> %mask, <8 x float> %op1, <8 x float> %op2
   store <8 x float> %sel, ptr %a
   ret void
 }

 ; CHECK-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x double> %op1, <2 x double> %op2
   ret <2 x double> %sel
 }
 
 define void @select_v4f64(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: select_v4f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    ldp q3, q2, [x1]
-; CHECK-NEXT:    fcmeq p1.d, p0/z, z1.d, z2.d
-; CHECK-NEXT:    fcmeq p0.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    fcmeq p1.d, p0/z, z0.d, z2.d
+; CHECK-NEXT:    fcmeq p0.d, p0/z, z1.d, z3.d
 ; CHECK-NEXT:    mov z4.d, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z5.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    bic z2.d, z2.d, z4.d
 ; CHECK-NEXT:    bic z3.d, z3.d, z5.d
-; CHECK-NEXT:    and z0.d, z0.d, z5.d
-; CHECK-NEXT:    and z1.d, z1.d, z4.d
-; CHECK-NEXT:    orr z0.d, z0.d, z3.d
-; CHECK-NEXT:    orr z1.d, z1.d, z2.d
-; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    and z1.d, z1.d, z5.d
+; CHECK-NEXT:    and z0.d, z0.d, z4.d
+; CHECK-NEXT:    orr z1.d, z1.d, z3.d
+; CHECK-NEXT:    orr z0.d, z0.d, z2.d
+; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %mask = fcmp oeq <4 x double> %op1, %op2
   %sel = select <4 x i1> %mask, <4 x double> %op1, <4 x double> %op2
   store <4 x double> %sel, ptr %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll

; CHECK-NEXT:    ret
     ret <2 x double> %r
 }
 
 define <4 x double> @insertelement_v4f64(ptr %a) #0 {
 ; CHECK-LABEL: insertelement_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    mov w8, #1
-; CHECK-NEXT:    fmov d3, #5.00000000
-; CHECK-NEXT:    index z4.d, #0, #1
+; CHECK-NEXT:    fmov d4, #5.00000000
+; CHECK-NEXT:    index z2.d, #0, #1
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z2.d, x8
-; CHECK-NEXT:    cmpeq p0.d, p0/z, z4.d, z2.d
-; CHECK-NEXT:    mov z1.d, p0/m, d3
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    cmpeq p0.d, p0/z, z2.d, z3.d
+; CHECK-NEXT:    mov z1.d, p0/m, d4
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
     %op1 = load <4 x double>, ptr %a
     %r = insertelement <4 x double> %op1, double 5.0, i64 3
     ret <4 x double> %r
 }
 
 attributes #0 = { "target-features"="+sve" }