[RISCV][AArch64][IRGen] Add scalable->fixed as a special case in CreateCoercedStore.

[craig.topper]

Summary

This improves the codegen for calling a function that returns a
VLST type. Previously we stored to an alloca using the scalable
type, but loaded it using the fixed vector type. The middle end is
unable to optimize away that store/load pair. With this patch we now
store using the fixed vector type which matches the load.

I have not added predicate types because I haven't supported
those on RISC-V yet so I haven't seen a problem yet.

[c-rhodes]

LGTM cheers

[c-rhodes]

It's been a while since I've looked at this code, what about this wouldnt work for predicates? I seem to recall fixed predicates using i8 vs i1 for scalable, is that issue? Happy with this regardless, just curious.

[craig.topper]

In D155222#4505008, @c-rhodes wrote:

It's been a while since I've looked at this code, what about this wouldnt work for predicates? I seem to recall fixed predicates using i8 vs i1 for scalable, is that issue? Happy with this regardless, just curious.

It probably would. I just have no experience with predicates yet.

[craig.topper]

I wonder if we should do something earlier CodeGenFunction::EmitCall before the call to CreateCoerecedStore and not go through memory at all?

[craig.topper]

Abandoning in favor of D155495

clang/lib/CodeGen/CGCall.cpp

 //===--- CGCall.cpp - Encapsulate calling convention details --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
static void CreateCoercedStore(llvm::Value *Src,
   // extension or truncation to the desired type.
   if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
       (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
     Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
     CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
     return;
   }
 
+  // If coercing a fixed vector from a scalable vector for ABI compatibility,
+  // and the types match, use the llvm.vector.extract intrinsic to perform the
+  // conversion.
+  if (auto *FixedDst = dyn_cast<llvm::FixedVectorType>(DstTy)) {
+    if (auto *ScalableSrc = dyn_cast<llvm::ScalableVectorType>(SrcTy)) {
+      if (FixedDst->getElementType() == ScalableSrc->getElementType()) {
+        auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
+        Src = CGF.Builder.CreateExtractVector(DstTy, Src, Zero, "cast.fixed");
+        CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
+        return;
+      }
+    }
+  }
   llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
 
   // If store is legal, just bitcast the src pointer.
   if (isa<llvm::ScalableVectorType>(SrcTy) ||
       isa<llvm::ScalableVectorType>(DstTy) ||
       SrcSize.getFixedValue() <= DstSize.getFixedValue()) {
     Dst = Dst.withElementType(SrcTy);
     CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);

clang/test/CodeGen/attr-arm-sve-vector-bits-call.c

 // CHECK-NEXT:    ret <vscale x 4 x i32> [[X_COERCE:%.*]]
 //
 fixed_int32_t fixed_callee(fixed_int32_t x) {
   return x;
 }
 
 // CHECK-LABEL: @sizeless_caller(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[COERCE1:%.*]] = alloca <16 x i32>, align 16
-// CHECK-NEXT:    store <vscale x 4 x i32> [[X:%.*]], ptr [[COERCE1]], align 16
-// CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr [[COERCE1]], align 16, !tbaa [[TBAA6:![0-9]+]]
-// CHECK-NEXT:    [[CASTSCALABLESVE2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[TMP1]], i64 0)
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE2]]
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[X:%.*]]
 //
 svint32_t sizeless_caller(svint32_t x) {
   return fixed_callee(x);
 }
 
 //===----------------------------------------------------------------------===//
 // fixed, fixed
 //===----------------------------------------------------------------------===//

clang/test/CodeGen/attr-riscv-rvv-vector-bits-call.c

 // CHECK-NEXT:    ret <vscale x 2 x i32> [[X_COERCE:%.*]]
 //
 fixed_int32m1_t fixed_callee(fixed_int32m1_t x) {
   return x;
 }
 
 // CHECK-LABEL: @sizeless_caller(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[COERCE1:%.*]] = alloca <8 x i32>, align 8
-// CHECK-NEXT:    store <vscale x 2 x i32> [[X:%.*]], ptr [[COERCE1]], align 8
-// CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[COERCE1]], align 8, !tbaa [[TBAA4:![0-9]+]]
-// CHECK-NEXT:    [[CASTSCALABLESVE2:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP0]], i64 0)
-// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE2]]
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[X:%.*]]
 //
 vint32m1_t sizeless_caller(vint32m1_t x) {
   return fixed_callee(x);
 }
 
 //===----------------------------------------------------------------------===//
 // fixed, fixed
 //===----------------------------------------------------------------------===//