diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -373,6 +373,20 @@ isScalable())); } + /// Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) + /// will result in a value whose size matches our own. + bool hasKnownScalarFactor(const LinearPolySize &RHS) const { + return isScalable() == RHS.isScalable() && + getKnownMinValue() % RHS.getKnownMinValue() == 0; + } + + /// Returns a value X where RHS.multiplyCoefficientBy(X) will result in a + /// value whose size matches our own. + ScalarTy getKnownScalarFactor(const LinearPolySize &RHS) const { + assert(hasKnownScalarFactor(RHS) && "Expected RHS to be a known factor!"); + return getKnownMinValue() / RHS.getKnownMinValue(); + } + /// Printing function. void print(raw_ostream &OS) const { if (isScalable()) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -453,13 +453,13 @@ // as the widened input type would be a legal type, we can widen the bitcast // and handle the promotion after. if (NOutVT.isVector()) { - unsigned WidenInSize = NInVT.getSizeInBits(); - unsigned OutSize = OutVT.getSizeInBits(); - if (WidenInSize % OutSize == 0) { - unsigned Scale = WidenInSize / OutSize; - EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(), - OutVT.getVectorElementType(), - OutVT.getVectorNumElements() * Scale); + TypeSize WidenInSize = NInVT.getSizeInBits(); + TypeSize OutSize = OutVT.getSizeInBits(); + if (WidenInSize.hasKnownScalarFactor(OutSize)) { + unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize); + EVT WideOutVT = + EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(), + OutVT.getVectorElementCount() * Scale); if (isTypeLegal(WideOutVT)) { InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, diff --git a/llvm/test/CodeGen/AArch64/sve-bitcast.ll b/llvm/test/CodeGen/AArch64/sve-bitcast.ll --- a/llvm/test/CodeGen/AArch64/sve-bitcast.ll +++ b/llvm/test/CodeGen/AArch64/sve-bitcast.ll @@ -518,6 +518,15 @@ ret %bc } +define @bitcast_nxv1i64_to_nxv8i8( %v) #0 { +; CHECK-LABEL: bitcast_nxv1i64_to_nxv8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv4f16_to_nxv8i8( %v) #0 { ; CHECK-LABEL: bitcast_nxv4f16_to_nxv8i8: ; CHECK: // %bb.0: @@ -550,6 +559,15 @@ ret %bc } +define @bitcast_nxv1f64_to_nxv8i8( %v) #0 { +; CHECK-LABEL: bitcast_nxv1f64_to_nxv8i8: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv4bf16_to_nxv8i8( %v) #0 { ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv8i8: ; CHECK: // %bb.0: @@ -602,6 +620,15 @@ ret %bc } +define @bitcast_nxv1i64_to_nxv4i16( %v) #0 { +; CHECK-LABEL: bitcast_nxv1i64_to_nxv4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv4f16_to_nxv4i16( %v) #0 { ; CHECK-LABEL: bitcast_nxv4f16_to_nxv4i16: ; CHECK: // %bb.0: @@ -626,6 +653,15 @@ ret %bc } +define @bitcast_nxv1f64_to_nxv4i16( %v) #0 { +; CHECK-LABEL: bitcast_nxv1f64_to_nxv4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv4bf16_to_nxv4i16( %v) #0 { ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv4i16: ; CHECK: // %bb.0: @@ -670,6 +706,15 @@ ret %bc } +define @bitcast_nxv1i64_to_nxv2i32( %v) #0 { +; CHECK-LABEL: bitcast_nxv1i64_to_nxv2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv4f16_to_nxv2i32( %v) #0 { ; CHECK-LABEL: bitcast_nxv4f16_to_nxv2i32: ; CHECK: // %bb.0: @@ -694,6 +739,15 @@ ret %bc } +define @bitcast_nxv1f64_to_nxv2i32( %v) #0 { +; CHECK-LABEL: bitcast_nxv1f64_to_nxv2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv4bf16_to_nxv2i32( %v) #0 { ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv2i32: ; CHECK: // %bb.0: @@ -710,6 +764,72 @@ ret %bc } +; +; bitcast to nxv1i64 +; + +define @bitcast_nxv8i8_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv8i8_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv4i16_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv4i16_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv2i32_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv2i32_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv4f16_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv4f16_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv2f32_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv2f32_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv1f64_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv1f64_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv4bf16_to_nxv1i64( %v) #0 { +; CHECK-LABEL: bitcast_nxv4bf16_to_nxv1i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + ; ; bitcast to nxv4f16 ; @@ -754,6 +874,8 @@ ret %bc } +; @bitcast_nxv1i64_to_nxv4f16 is missing + define @bitcast_nxv2f32_to_nxv4f16( %v) #0 { ; CHECK-LABEL: bitcast_nxv2f32_to_nxv4f16: ; CHECK: // %bb.0: @@ -770,6 +892,8 @@ ret %bc } +; @bitcast_nxv1f64_to_nxv4f16 is missing + define @bitcast_nxv4bf16_to_nxv4f16( %v) #0 { ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv4f16: ; CHECK: // %bb.0: @@ -822,6 +946,8 @@ ret %bc } +; @bitcast_nxv1i64_to_nxv2f32 is missing + define @bitcast_nxv4f16_to_nxv2f32( %v) #0 { ; CHECK-LABEL: bitcast_nxv4f16_to_nxv2f32: ; CHECK: // %bb.0: @@ -838,6 +964,8 @@ ret %bc } +; @bitcast_nxv1f64_to_nxv2f32 is missing + define @bitcast_nxv4bf16_to_nxv2f32( %v) #0 { ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv2f32: ; CHECK: // %bb.0: @@ -854,6 +982,18 @@ ret %bc } +; +; bitcast to nxv1f64 +; + +; @bitcast_nxv8i8_to_nxv1f64 is missing +; @bitcast_nxv4i16_to_nxv1f64 is missing +; @bitcast_nxv2i32_to_nxv1f64 is missing +; @bitcast_nxv1i64_to_nxv1f64 is missing +; @bitcast_nxv4f16_to_nxv1f64 is missing +; @bitcast_nxv2f32_to_nxv1f64 is missing +; @bitcast_nxv4bf16_to_nxv1f64 is missing + ; ; bitcast to nxv4bf16 ; @@ -898,6 +1038,8 @@ ret %bc } +; @bitcast_nxv1i64_to_nxv4bf16 is missing + define @bitcast_nxv4f16_to_nxv4bf16( %v) #0 { ; CHECK-LABEL: bitcast_nxv4f16_to_nxv4bf16: ; CHECK: // %bb.0: @@ -922,6 +1064,8 @@ ret %bc } +; @bitcast_nxv1f64_to_nxv4bf16 is missing + ; ; bitcast to nxv4i8 ; @@ -942,6 +1086,16 @@ ret %bc } +define @bitcast_nxv1i32_to_nxv4i8( %v) #0 { +; CHECK-LABEL: bitcast_nxv1i32_to_nxv4i8: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv2f16_to_nxv4i8( %v) #0 { ; CHECK-LABEL: bitcast_nxv2f16_to_nxv4i8: ; CHECK: // %bb.0: @@ -958,6 +1112,8 @@ ret %bc } +; @bitcast_nxv1f32_to_nxv4i8 is missing + define @bitcast_nxv2bf16_to_nxv4i8( %v) #0 { ; CHECK-LABEL: bitcast_nxv2bf16_to_nxv4i8: ; CHECK: // %bb.0: @@ -994,6 +1150,16 @@ ret %bc } +define @bitcast_nxv1i32_to_nxv2i16( %v) #0 { +; CHECK-LABEL: bitcast_nxv1i32_to_nxv2i16: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + define @bitcast_nxv2f16_to_nxv2i16( %v) #0 { ; CHECK-LABEL: bitcast_nxv2f16_to_nxv2i16: ; CHECK: // %bb.0: @@ -1002,6 +1168,8 @@ ret %bc } +; @bitcast_nxv1f32_to_nxv2i16 is missing + define @bitcast_nxv2bf16_to_nxv2i16( %v) #0 { ; CHECK-LABEL: bitcast_nxv2bf16_to_nxv2i16: ; CHECK: // %bb.0: @@ -1010,6 +1178,64 @@ ret %bc } +; +; bitcast to nxv1i32 +; + +define @bitcast_nxv4i8_to_nxv1i32( %v) #0 { +; CHECK-LABEL: bitcast_nxv4i8_to_nxv1i32: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv2i16_to_nxv1i32( %v) #0 { +; CHECK-LABEL: bitcast_nxv2i16_to_nxv1i32: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +define @bitcast_nxv2f16_to_nxv1i32( %v) #0 { +; CHECK-LABEL: bitcast_nxv2f16_to_nxv1i32: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: st1h { z0.d }, p0, [sp] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] +; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +; @bitcast_nxv1f32_to_nxv1i32 is missing + +define @bitcast_nxv2bf16_to_nxv1i32( %v) #0 { +; CHECK-LABEL: bitcast_nxv2bf16_to_nxv1i32: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: st1h { z0.d }, p0, [sp] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] +; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + ; ; bitcast to nxv2f16 ; @@ -1038,6 +1264,9 @@ ret %bc } +; @bitcast_nxv1i32_to_nxv2f16 is missing +; @bitcast_nxv1f32_to_nxv2f16 is missing + define @bitcast_nxv2bf16_to_nxv2f16( %v) #0 { ; CHECK-LABEL: bitcast_nxv2bf16_to_nxv2f16: ; CHECK: // %bb.0: @@ -1074,6 +1303,8 @@ ret %bc } +; @bitcast_nxv1i32_to_nxv2bf16 is missing + define @bitcast_nxv2f16_to_nxv2bf16( %v) #0 { ; CHECK-LABEL: bitcast_nxv2f16_to_nxv2bf16: ; CHECK: // %bb.0: @@ -1082,6 +1313,44 @@ ret %bc } +; @bitcast_nxv1f32_to_nxv2bf16 is missing + +; +; bitcast to nxv2i8 +; + +define @bitcast_nxv1i16_to_nxv2i8( %v) #0 { +; CHECK-LABEL: bitcast_nxv1i16_to_nxv2i8: +; CHECK: // %bb.0: +; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +; @bitcast_nxv1f16_to_nxv2i8 is missing +; @bitcast_nxv1bf16_to_nxv2i8 is missing + +; +; bitcast to nxv1i16 +; + +define @bitcast_nxv2i8_to_nxv1i16( %v) #0 { +; CHECK-LABEL: bitcast_nxv2i8_to_nxv1i16: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK-NEXT: ret + %bc = bitcast %v to + ret %bc +} + +; @bitcast_nxv1f16_to_nxv1i16 is missing +; @bitcast_nxv1bf16_to_nxv1i16 is missing + ; ; Other ;