[SVE] Fix incorrect code generation for bitcasts of unpacked vector types.

[paulwalker-arm]

Summary

Bitcasting between unpacked scalable vector types of different
element counts is not a NOP because the live elements are laid out
differently.

01234567

e.g. nxv2i32 = XX??XX??

nxv4f16 = X?X?X?X?

[paulwalker-arm]

For this patch I'm just playing it safe and using common expansion via the stack rather than anything more exotic.

[efriedma]

The "correct" expansion isn't too tricky. First, compute "packed" types for the source and destination. Then just insert_subvector into the packed source, bitcast to the packed destination, then extract_subvector. Should optimize to one or two instructions once we lower the insert_subvector/extract_subvector into shuffles. Not that I'm really against the current version, but if we're going to end up rewriting this code anyway, might as well do it now.

Should we add an assertion to getSVESafeBitCast() that we aren't performing a bitcast like this?

[paulwalker-arm]

@efriedma I'd still rather have the improved code generation as a separate patch, mainly because this patch forces the need to remove an invalid type size conversion from SelectionDAGLegalize::EmitStackConvert and I'd like there to be a window (however small) that validates the code now works for fixed and scalable vector types.

[paulwalker-arm]

Add assert to getSVESafeBitCast to detect original failure case.

[efriedma]

LGTM

llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
                                                EVT DestVT, const SDLoc &dl) {
   return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
 }
 
 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
                                                EVT DestVT, const SDLoc &dl,
                                                SDValue Chain) {
-  unsigned SrcSize = SrcOp.getValueSizeInBits();
-  unsigned SlotSize = SlotVT.getSizeInBits();
-  unsigned DestSize = DestVT.getSizeInBits();
+  EVT SrcVT = SrcOp.getValueType();
   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
   Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
 
   // Don't convert with stack if the load/store is expensive.
-  if ((SrcSize > SlotSize &&
+  if ((SrcVT.bitsGT(SlotVT) &&
        !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
-      (SlotSize < DestSize &&
+      (SlotVT.bitsLT(DestVT) &&
        !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
     return SDValue();
 
   // Create the stack frame object.
   Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
       SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
 
   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
   int SPFI = StackPtrFI->getIndex();
   MachinePointerInfo PtrInfo =
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
 
   // Emit a store to the stack slot.  Use a truncstore if the input value is
   // later than DestVT.
   SDValue Store;
 
-  if (SrcSize > SlotSize)
+  if (SrcVT.bitsGT(SlotVT))
     Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
                               SlotVT, SrcAlign);
   else {
-    assert(SrcSize == SlotSize && "Invalid store");
-    Store =
-        DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
+    assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
+    Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
   }
 
   // Result is a load from the stack slot.
-  if (SlotSize == DestSize)
+  if (SlotVT.bitsEq(DestVT))
     return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
 
-  assert(SlotSize < DestSize && "Unknown extension!");
+  assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
   return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
                         DestAlign);
 }
 
 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
   SDLoc dl(Node);
   // Create a vector sized/aligned stack slot, store the value to element #0,
   // then load the whole vector back out.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

SDValue AArch64TargetLowering::LowerBITCAST(SDValue Op,
                                             SelectionDAG &DAG) const {
   EVT OpVT = Op.getValueType();
   EVT ArgVT = Op.getOperand(0).getValueType();
 
   if (useSVEForFixedLengthVectorVT(OpVT))
     return LowerFixedLengthBitcastToSVE(Op, DAG);
 
   if (OpVT.isScalableVector()) {
+    // Bitcasting between unpacked vector types of different element counts is
+    // not a NOP because the live elements are laid out differently.
+    //                01234567
+    // e.g. nxv2i32 = XX??XX??
+    //      nxv4f16 = X?X?X?X?
+    if (OpVT.getVectorElementCount() != ArgVT.getVectorElementCount())
+      return SDValue();
+
     if (isTypeLegal(OpVT) && !isTypeLegal(ArgVT)) {
       assert(OpVT.isFloatingPoint() && !ArgVT.isFloatingPoint() &&
              "Expected int->fp bitcast!");
       SDValue ExtResult =
           DAG.getNode(ISD::ANY_EXTEND, SDLoc(Op), getSVEContainerType(ArgVT),
                       Op.getOperand(0));
       return getSVESafeBitCast(OpVT, ExtResult, DAG);
     }
void AArch64TargetLowering::ReplaceBITCASTResults(
   SDLoc DL(N);
   SDValue Op = N->getOperand(0);
   EVT VT = N->getValueType(0);
   EVT SrcVT = Op.getValueType();
 
   if (VT.isScalableVector() && !isTypeLegal(VT) && isTypeLegal(SrcVT)) {
     assert(!VT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
            "Expected fp->int bitcast!");
+
+    // Bitcasting between unpacked vector types of different element counts is
+    // not a NOP because the live elements are laid out differently.
+    //                01234567
+    // e.g. nxv2i32 = XX??XX??
+    //      nxv4f16 = X?X?X?X?
+    if (VT.getVectorElementCount() != SrcVT.getVectorElementCount())
+      return;
+
     SDValue CastResult = getSVESafeBitCast(getSVEContainerType(VT), Op, DAG);
     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, CastResult));
     return;
   }
 
   if (VT != MVT::i16 || (SrcVT != MVT::f16 && SrcVT != MVT::bf16))
     return;
 
if (InVT == VT)
     return Op;
 
   if (VT.getVectorElementType() == MVT::i1)
     return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
 
   EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType());
   EVT PackedInVT = getPackedSVEVectorVT(InVT.getVectorElementType());
 
+  // Safe bitcasting between unpacked vector types of different element counts
+  // is currently unsupported because the following is missing the necessary
+  // work to ensure the result's elements live where they're supposed to within
+  // an SVE register.
+  //                01234567
+  // e.g. nxv2i32 = XX??XX??
+  //      nxv4f16 = X?X?X?X?
+  assert((VT.getVectorElementCount() == InVT.getVectorElementCount() ||
+          VT == PackedVT || InVT == PackedInVT) &&
+         "Unexpected bitcast!");
+
   // Pack input if required.
   if (InVT != PackedInVT)
     Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, PackedInVT, Op);
 
   Op = DAG.getNode(ISD::BITCAST, DL, PackedVT, Op);
 
   // Unpack result if required.
   if (VT != PackedVT)

llvm/test/CodeGen/AArch64/sve-bitcast.ll

 define <vscale x 4 x i16> @bitcast_nxv4f16_to_nxv4i16(<vscale x 4 x half> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4f16_to_nxv4i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x half> %v to <vscale x 4 x i16>
   ret <vscale x 4 x i16> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 4 x i16> @bitcast_nxv2f32_to_nxv4i16(<vscale x 2 x float> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2f32_to_nxv4i16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x float> %v to <vscale x 4 x i16>
   ret <vscale x 4 x i16> %bc
 }
 
 define <vscale x 4 x i16> @bitcast_nxv4bf16_to_nxv4i16(<vscale x 4 x bfloat> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv4i16:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x i16> %v to <vscale x 2 x i32>
   ret <vscale x 2 x i32> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 2 x i32> @bitcast_nxv4f16_to_nxv2i32(<vscale x 4 x half> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4f16_to_nxv2i32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x half> %v to <vscale x 2 x i32>
   ret <vscale x 2 x i32> %bc
 }
 
 define <vscale x 2 x i32> @bitcast_nxv2f32_to_nxv2i32(<vscale x 2 x float> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2f32_to_nxv2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x float> %v to <vscale x 2 x i32>
   ret <vscale x 2 x i32> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 2 x i32> @bitcast_nxv4bf16_to_nxv2i32(<vscale x 4 x bfloat> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv2i32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x bfloat> %v to <vscale x 2 x i32>
   ret <vscale x 2 x i32> %bc
 }
 
 ;
 ; bitcast to nxv4f16
 ;

 define <vscale x 4 x half> @bitcast_nxv4i16_to_nxv4f16(<vscale x 4 x i16> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4i16_to_nxv4f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x i16> %v to <vscale x 4 x half>
   ret <vscale x 4 x half> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 4 x half> @bitcast_nxv2i32_to_nxv4f16(<vscale x 2 x i32> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2i32_to_nxv4f16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x i32> %v to <vscale x 4 x half>
   ret <vscale x 4 x half> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 4 x half> @bitcast_nxv2f32_to_nxv4f16(<vscale x 2 x float> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2f32_to_nxv4f16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x float> %v to <vscale x 4 x half>
   ret <vscale x 4 x half> %bc
 }
 
 define <vscale x 4 x half> @bitcast_nxv4bf16_to_nxv4f16(<vscale x 4 x bfloat> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv4f16:
 ; CHECK:       // %bb.0:

 ; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 8 x i8> %v to <vscale x 2 x float>
   ret <vscale x 2 x float> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 2 x float> @bitcast_nxv4i16_to_nxv2f32(<vscale x 4 x i16> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4i16_to_nxv2f32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x i16> %v to <vscale x 2 x float>
   ret <vscale x 2 x float> %bc
 }
 
 define <vscale x 2 x float> @bitcast_nxv2i32_to_nxv2f32(<vscale x 2 x i32> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2i32_to_nxv2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x i32> %v to <vscale x 2 x float>
   ret <vscale x 2 x float> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 2 x float> @bitcast_nxv4f16_to_nxv2f32(<vscale x 4 x half> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4f16_to_nxv2f32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x half> %v to <vscale x 2 x float>
   ret <vscale x 2 x float> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 2 x float> @bitcast_nxv4bf16_to_nxv2f32(<vscale x 4 x bfloat> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4bf16_to_nxv2f32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x bfloat> %v to <vscale x 2 x float>
   ret <vscale x 2 x float> %bc
 }
 
 ;
 ; bitcast to nxv4bf16
 ;

 define <vscale x 4 x bfloat> @bitcast_nxv4i16_to_nxv4bf16(<vscale x 4 x i16> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4i16_to_nxv4bf16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x i16> %v to <vscale x 4 x bfloat>
   ret <vscale x 4 x bfloat> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 4 x bfloat> @bitcast_nxv2i32_to_nxv4bf16(<vscale x 2 x i32> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2i32_to_nxv4bf16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x i32> %v to <vscale x 4 x bfloat>
   ret <vscale x 4 x bfloat> %bc
 }
 
 define <vscale x 4 x bfloat> @bitcast_nxv4f16_to_nxv4bf16(<vscale x 4 x half> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv4f16_to_nxv4bf16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 4 x half> %v to <vscale x 4 x bfloat>
   ret <vscale x 4 x bfloat> %bc
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 4 x bfloat> @bitcast_nxv2f32_to_nxv4bf16(<vscale x 2 x float> %v) #0 {
 ; CHECK-LABEL: bitcast_nxv2f32_to_nxv4bf16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %bc = bitcast <vscale x 2 x float> %v to <vscale x 4 x bfloat>
   ret <vscale x 4 x bfloat> %bc
 }
 
 ;
 ; bitcast to nxv4i8
 ;

 ; CHECK-NEXT:    fcvt z0.d, p0/m, z0.s
 ; CHECK-NEXT:    ret
   %trunc = trunc <vscale x 2 x i64> %v to <vscale x 2 x i32>
   %bitcast = bitcast <vscale x 2 x i32> %trunc to <vscale x 2 x float>
   %extended = fpext <vscale x 2 x float> %bitcast to <vscale x 2 x double>
   ret <vscale x 2 x double> %extended
 }
 
-; TODO: Invalid code generation because the bitcast must change the in-register
-; layout when casting between unpacked scalable vector types.
 define <vscale x 2 x float> @bitcast_short_half_to_float(<vscale x 4 x half> %v) #0 {
 ; CHECK-LABEL: bitcast_short_half_to_float:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z0.h
+; CHECK-NEXT:    st1h { z0.s }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %add = fadd <vscale x 4 x half> %v, %v
   %bitcast = bitcast <vscale x 4 x half> %add to <vscale x 2 x float>
   ret <vscale x 2 x float> %bitcast
 }
 
 ; +bf16 is required for the bfloat version.
 attributes #0 = { nounwind "target-features"="+sve,+bf16" }