diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h --- a/llvm/include/llvm/IR/DataLayout.h +++ b/llvm/include/llvm/IR/DataLayout.h @@ -620,16 +620,16 @@ /// Used to lazily calculate structure layout information for a target machine, /// based on the DataLayout structure. class StructLayout { - uint64_t StructSize; + TypeSize StructSize; Align StructAlignment; unsigned IsPadded : 1; unsigned NumElements : 31; - uint64_t MemberOffsets[1]; // variable sized array! + SmallVector MemberOffsets; public: - uint64_t getSizeInBytes() const { return StructSize; } + TypeSize getSizeInBytes() const { return StructSize; } - uint64_t getSizeInBits() const { return 8 * StructSize; } + TypeSize getSizeInBits() const { return 8 * StructSize; } Align getAlignment() const { return StructAlignment; } @@ -643,7 +643,9 @@ uint64_t getElementOffset(unsigned Idx) const { assert(Idx < NumElements && "Invalid element idx!"); - return MemberOffsets[Idx]; + if (StructSize.isScalable()) + return MemberOffsets[Idx].getScalable(); + return MemberOffsets[Idx].getFixed(); } uint64_t getElementOffsetInBits(unsigned Idx) const { @@ -672,8 +674,7 @@ } case Type::StructTyID: // Get the layout annotation... which is lazily created on demand. - return TypeSize::Fixed( - getStructLayout(cast(Ty))->getSizeInBits()); + return getStructLayout(cast(Ty))->getSizeInBits(); case Type::IntegerTyID: return TypeSize::Fixed(Ty->getIntegerBitWidth()); case Type::HalfTyID: diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h --- a/llvm/include/llvm/IR/Type.h +++ b/llvm/include/llvm/IR/Type.h @@ -236,6 +236,9 @@ return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID; } + /// True if this is an instance of scalable types. + bool isScalableType() const; + /// Return true if this type could be converted with a lossless BitCast to /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the /// same size only where no re-interpretation of the bits is done. diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -1003,6 +1003,7 @@ NewIdxs.push_back(ConstantInt::get(IntIdxTy, NewIdx)); } } else { + // TODO: Review for scalable struct. auto *STy = cast(Ty); // If we end up with an offset that isn't valid for this struct type, we // can't re-form this GEP in a regular form, so bail out. The pointer diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp --- a/llvm/lib/Analysis/MemoryBuiltins.cpp +++ b/llvm/lib/Analysis/MemoryBuiltins.cpp @@ -355,6 +355,7 @@ return nullptr; unsigned ElementSize = DL.getTypeAllocSize(T); + // TODO: Review for scalable struct. if (StructType *ST = dyn_cast(T)) ElementSize = DL.getStructLayout(ST)->getSizeInBytes(); diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3442,6 +3442,7 @@ SmallVector Offsets; for (const SCEV *IndexExpr : IndexExprs) { // Compute the (potentially symbolic) offset in bytes for this index. + // TODO: Review for scalable struct. if (StructType *STy = dyn_cast(CurTy)) { // For a struct, add the member offset. ConstantInt *Index = cast(IndexExpr)->getValue(); diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp --- a/llvm/lib/CodeGen/Analysis.cpp +++ b/llvm/lib/CodeGen/Analysis.cpp @@ -85,6 +85,7 @@ SmallVectorImpl *Offsets, uint64_t StartingOffset) { // Given a struct type, recursively traverse the elements. + // TODO: Review for scalable struct. if (StructType *STy = dyn_cast(Ty)) { // If the Offsets aren't needed, don't query the struct layout. This allows // us to support structs with scalable vectors for operations that don't @@ -134,6 +135,7 @@ SmallVectorImpl *Offsets, uint64_t StartingOffset) { // Given a struct type, recursively traverse the elements. + // TODO: Review for scalable struct. if (StructType *STy = dyn_cast(&Ty)) { // If the Offsets aren't needed, don't query the struct layout. This allows // us to support structs with scalable vectors for operations that don't diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -174,9 +174,12 @@ // Scalable vectors may need a special StackID to distinguish // them from other (fixed size) stack objects. - if (isa(Ty)) + if (Ty->isScalableType()) { + assert(Ty->isSized() && "It is only permitted to alloca struct " + "with the same scalable vector types."); MF->getFrameInfo().setStackID(FrameIndex, TFI->getStackIDForScalableVectors()); + } StaticAllocaMap[AI] = FrameIndex; // Update the catch handler information. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4079,7 +4079,6 @@ SmallVector Values(NumValues); SmallVector Chains(std::min(MaxParallelChains, NumValues)); - EVT PtrVT = Ptr.getValueType(); MachineMemOperand::Flags MMOFlags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); @@ -4099,10 +4098,12 @@ Root = Chain; ChainI = 0; } - SDValue A = DAG.getNode(ISD::ADD, dl, - PtrVT, Ptr, - DAG.getConstant(Offsets[i], dl, PtrVT), - Flags); + SDValue A; + if (ValueVTs[i].isScalableVector() && Offsets[i] != 0) + A = DAG.getMemBasePlusOffset(Ptr, TypeSize::Scalable(Offsets[i]), dl, + Flags); + else + A = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, MachinePointerInfo(SV, Offsets[i]), Alignment, @@ -4245,8 +4246,13 @@ Root = Chain; ChainI = 0; } - SDValue Add = - DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); + SDValue Add; + if (ValueVTs[i].isScalableVector() && Offsets[i] != 0) + Add = DAG.getMemBasePlusOffset(Ptr, TypeSize::Scalable(Offsets[i]), dl, + Flags); + else + Add = + DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); if (MemVTs[i] != ValueVTs[i]) Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp --- a/llvm/lib/IR/DataLayout.cpp +++ b/llvm/lib/IR/DataLayout.cpp @@ -44,11 +44,16 @@ // Support for StructLayout //===----------------------------------------------------------------------===// -StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { +StructLayout::StructLayout(StructType *ST, const DataLayout &DL) + : StructSize(0, false) { assert(!ST->isOpaque() && "Cannot get layout of opaque structs"); - StructSize = 0; + // We permit scalable vector struct as "sized" only when all the elements + // are the same scalable vector types. + if (ST->isScalableType()) + StructSize = TypeSize::Scalable(0); IsPadded = false; NumElements = ST->getNumElements(); + StackOffset Offset = StackOffset::getFixed(0); // Loop over each of the elements, placing them in memory. for (unsigned i = 0, e = NumElements; i != e; ++i) { @@ -56,37 +61,70 @@ const Align TyAlign = ST->isPacked() ? Align(1) : DL.getABITypeAlign(Ty); // Add padding if necessary to align the data element properly. - if (!isAligned(TyAlign, StructSize)) { + // Scalable vector struct must be the same scalable vector types. They have + // no alignment issues. + if (!StructSize.isScalable() && + !isAligned(TyAlign, StructSize.getFixedSize())) { IsPadded = true; - StructSize = alignTo(StructSize, TyAlign); + uint64_t Padding = alignTo(StructSize.getFixedSize(), TyAlign) - + StructSize.getFixedSize(); + Offset += StackOffset::getFixed(Padding); + StructSize = TypeSize::Fixed(alignTo(StructSize.getFixedSize(), TyAlign)); } // Keep track of maximum alignment constraint. StructAlignment = std::max(TyAlign, StructAlignment); - MemberOffsets[i] = StructSize; + MemberOffsets.push_back(Offset); // Consume space for this data item - StructSize += DL.getTypeAllocSize(Ty).getFixedValue(); + TypeSize ElemSize = DL.getTypeAllocSize(Ty); + StructSize += ElemSize; + if (Ty->isScalableType()) + Offset += StackOffset::getScalable(ElemSize.getKnownMinSize()); + else + Offset += StackOffset::getFixed(ElemSize.getFixedSize()); } // Add padding to the end of the struct so that it could be put in an array // and all array elements would be aligned correctly. - if (!isAligned(StructAlignment, StructSize)) { + if (!StructSize.isScalable() && + !isAligned(StructAlignment, StructSize.getFixedSize())) { IsPadded = true; - StructSize = alignTo(StructSize, StructAlignment); + StructSize = + TypeSize::Fixed(alignTo(StructSize.getFixedSize(), StructAlignment)); } } +// The struct either contains scalar elements or scalable elements. +// It could not mix scalar and scalable types within the struct at the same +// time. +static bool isKnownLT(StackOffset lhs, StackOffset rhs) { + return (lhs.getFixed() < rhs.getFixed()) || + (lhs.getScalable() < rhs.getScalable()); +} + +static bool isKnownLE(StackOffset lhs, StackOffset rhs) { + if (lhs.getFixed() != 0) + return lhs.getFixed() <= rhs.getFixed(); + return lhs.getScalable() <= rhs.getScalable(); +} + /// getElementContainingOffset - Given a valid offset into the structure, /// return the structure index that contains it. -unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const { - const uint64_t *SI = - std::upper_bound(&MemberOffsets[0], &MemberOffsets[NumElements], Offset); - assert(SI != &MemberOffsets[0] && "Offset not in structure type!"); +unsigned StructLayout::getElementContainingOffset(uint64_t O) const { + StackOffset Offset = StackOffset::getFixed(O); + if (StructSize.isScalable()) + Offset = StackOffset::getScalable(O); + const StackOffset *SI = + std::upper_bound(MemberOffsets.begin(), MemberOffsets.end(), Offset, + [](StackOffset lhs, StackOffset rhs) -> bool { + return isKnownLT(lhs, rhs); + }); + assert(SI != MemberOffsets.begin() && "Offset not in structure type!"); --SI; - assert(*SI <= Offset && "upper_bound didn't work"); - assert((SI == &MemberOffsets[0] || *(SI-1) <= Offset) && - (SI+1 == &MemberOffsets[NumElements] || *(SI+1) > Offset) && + assert(isKnownLE(*SI, Offset) && "upper_bound didn't work"); + assert((SI == MemberOffsets.begin() || isKnownLE(*(SI - 1), Offset)) && + (SI + 1 == MemberOffsets.end() || isKnownLT(Offset, *(SI + 1))) && "Upper bound didn't work!"); // Multiple fields can have the same offset if any of them are zero sized. @@ -94,7 +132,7 @@ // at the i32 element, because it is the last element at that offset. This is // the right one to return, because anything after it will have a higher // offset, implying that this element is non-empty. - return SI-&MemberOffsets[0]; + return SI - MemberOffsets.begin(); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -180,6 +180,17 @@ return cast(this)->isSized(Visited); } +bool Type::isScalableType() const { + if (getTypeID() == ScalableVectorTyID) + return true; + + if (auto *STy = dyn_cast(this)) { + if (STy->containsScalableVectorType()) + return true; + } + return false; +} + //===----------------------------------------------------------------------===// // Primitive 'Type' data //===----------------------------------------------------------------------===// @@ -528,11 +539,22 @@ // Okay, our struct is sized if all of the elements are, but if one of the // elements is opaque, the struct isn't sized *yet*, but may become sized in // the future, so just bail out without caching. + Type *FirstTy = getNumElements() > 0 ? elements()[0] : nullptr; + bool IsFirstOneScalable = false; + if (FirstTy) + IsFirstOneScalable = isa(FirstTy); for (Type *Ty : elements()) { - // If the struct contains a scalable vector type, don't consider it sized. - // This prevents it from being used in loads/stores/allocas/GEPs. - if (isa(Ty)) - return false; + if (IsFirstOneScalable) { + // We do not permit mix scalar types with scalable types within struct. + if (!isa(Ty)) + return false; + // All the scalable types within struct should be the same. + if (FirstTy != Ty) + return false; + } else { + if (isa(Ty)) + return false; + } if (!Ty->isSized(Visited)) return false; } diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp --- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -797,6 +797,7 @@ return true; // Check for padding within and between elements of a struct. + // TODO: Review for scalable struct. StructType *StructTy = cast(type); const StructLayout *Layout = DL.getStructLayout(StructTy); uint64_t StartPos = 0; diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -213,6 +213,7 @@ // Non-aggregate type, we cast and make byte-wise progress now. break; + // TODO: Review for scalable struct. const StructLayout *SL = DL.getStructLayout(STy); if (int64_t(SL->getSizeInBytes()) < Offset) break; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -628,6 +628,7 @@ StringRef Name = LI.getName(); assert(LI.getAlignment() && "Alignment must be set at this point"); + // TODO: Review for scalable struct. if (auto *ST = dyn_cast(T)) { // If the struct only have one element, we unpack. auto NumElements = ST->getNumElements(); @@ -1157,6 +1158,7 @@ if (!T->isAggregateType()) return false; + // TODO: Review for scalable struct. if (auto *ST = dyn_cast(T)) { // If the struct only have one element, we unpack. unsigned Count = ST->getNumElements(); diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1260,6 +1260,7 @@ return nullptr; if (StructType *STy = dyn_cast(Ty)) { + // TODO: Review for scalable struct. const StructLayout *SL = DL.getStructLayout(STy); assert(Offset < (int64_t)SL->getSizeInBytes() && "Offset must stay within the indexed type"); diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -1506,6 +1506,7 @@ Indices, NamePrefix); } + // TODO: Review for scalable struct. StructType *STy = dyn_cast(Ty); if (!STy) return nullptr; @@ -3620,6 +3621,7 @@ if (ArrayType *ArrTy = dyn_cast(Ty)) { InnerTy = ArrTy->getElementType(); } else if (StructType *STy = dyn_cast(Ty)) { + // TODO: Review for scalable struct. const StructLayout *SL = DL.getStructLayout(STy); unsigned Index = SL->getElementContainingOffset(0); InnerTy = STy->getElementType(Index); @@ -3693,6 +3695,7 @@ return ArrayType::get(ElementTy, NumElements); } + // TODO: Review for scalable struct. StructType *STy = dyn_cast(Ty); if (!STy) return nullptr; diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -498,6 +498,7 @@ break; if (const SCEVConstant *C = dyn_cast(Ops[0])) if (SE.getTypeSizeInBits(C->getType()) <= 64) { + // TODO: Review for scalable struct. const StructLayout &SL = *DL.getStructLayout(STy); uint64_t FullOffset = C->getValue()->getZExtValue(); if (FullOffset < SL.getSizeInBytes()) { diff --git a/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll @@ -0,0 +1,47 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-v -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s + +target triple = "riscv64-unknown-unknown-elf" + +%struct.test = type { , } + +define @test(%struct.test* %addr, i64 %vl) { +; CHECK-LABEL: test: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: csrrs a2, vlenb, zero +; CHECK-NEXT: slli a2, a2, 1 +; CHECK-NEXT: sub sp, sp, a2 +; CHECK-NEXT: csrrs a2, vlenb, zero +; CHECK-NEXT: add a3, a0, a2 +; CHECK-NEXT: vl1re64.v v25, (a3) +; CHECK-NEXT: vl1re64.v v26, (a0) +; CHECK-NEXT: addi a0, sp, 0 +; CHECK-NEXT: add a0, a0, a2 +; CHECK-NEXT: vs1r.v v25, (a0) +; CHECK-NEXT: vs1r.v v26, (sp) +; CHECK-NEXT: vl1re64.v v25, (a0) +; CHECK-NEXT: vl1re64.v v26, (sp) +; CHECK-NEXT: vsetvli a0, a1, e64,m1,ta,mu +; CHECK-NEXT: vfadd.vv v8, v26, v25 +; CHECK-NEXT: csrrs a0, vlenb, zero +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: add sp, sp, a0 +; CHECK-NEXT: jalr zero, 0(ra) +entry: + %ret = alloca %struct.test, align 8 + %val = load %struct.test, %struct.test* %addr + store %struct.test %val, %struct.test* %ret, align 8 + %0 = load %struct.test, %struct.test* %ret, align 8 + %1 = extractvalue %struct.test %0, 0 + %2 = extractvalue %struct.test %0, 1 + %3 = call @llvm.riscv.vfadd.nxv1f64.nxv1f64( + %1, %2, i64 %vl) + ret %3 +} + +declare @llvm.riscv.vfadd.nxv1f64.nxv1f64( + , + , + i64); diff --git a/llvm/test/Other/load-scalable-vector-struct.ll b/llvm/test/Other/load-scalable-vector-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Other/load-scalable-vector-struct.ll @@ -0,0 +1,12 @@ +; RUN: opt -S -verify < %s 2>&1 | FileCheck %s + +%struct.test = type { , } + +define @load(%struct.test* %x) { +; CHECK: %a = load %struct.test, %struct.test* %x, align 4 +; CHECK: %b = extractvalue %struct.test %a, 1 +; CHECK: ret %b + %a = load %struct.test, %struct.test* %x + %b = extractvalue %struct.test %a, 1 + ret %b +} diff --git a/llvm/test/Other/store-scalable-vector-struct.ll b/llvm/test/Other/store-scalable-vector-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Other/store-scalable-vector-struct.ll @@ -0,0 +1,14 @@ +; RUN: opt -S -verify < %s 2>&1 | FileCheck %s + +%struct.test = type { , } + +define void @store(%struct.test* %x, %y, %z) { +; CHECK: %a = insertvalue %struct.test undef, %y, 0 +; CHECK: %b = insertvalue %struct.test %a, %z, 1 +; CHECK: store %struct.test %b, %struct.test* %x +; CHECK: ret void + %a = insertvalue %struct.test undef, %y, 0 + %b = insertvalue %struct.test %a, %z, 1 + store %struct.test %b, %struct.test* %x + ret void +}