diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h --- a/llvm/include/llvm/IR/DataLayout.h +++ b/llvm/include/llvm/IR/DataLayout.h @@ -620,16 +620,16 @@ /// Used to lazily calculate structure layout information for a target machine, /// based on the DataLayout structure. class StructLayout { - uint64_t StructSize; + TypeSize StructSize; Align StructAlignment; unsigned IsPadded : 1; unsigned NumElements : 31; - uint64_t MemberOffsets[1]; // variable sized array! + SmallVector MemberOffsets; public: - uint64_t getSizeInBytes() const { return StructSize; } + uint64_t getSizeInBytes() const { return StructSize.getKnownMinValue(); } - uint64_t getSizeInBits() const { return 8 * StructSize; } + TypeSize getSizeInBits() const { return 8 * StructSize; } Align getAlignment() const { return StructAlignment; } @@ -643,7 +643,9 @@ uint64_t getElementOffset(unsigned Idx) const { assert(Idx < NumElements && "Invalid element idx!"); - return MemberOffsets[Idx]; + if (StructSize.isScalable()) + return MemberOffsets[Idx].getScalable(); + return MemberOffsets[Idx].getFixed(); } uint64_t getElementOffsetInBits(unsigned Idx) const { @@ -672,8 +674,7 @@ } case Type::StructTyID: // Get the layout annotation... which is lazily created on demand. - return TypeSize::Fixed( - getStructLayout(cast(Ty))->getSizeInBits()); + return getStructLayout(cast(Ty))->getSizeInBits(); case Type::IntegerTyID: return TypeSize::Fixed(Ty->getIntegerBitWidth()); case Type::HalfTyID: diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h --- a/llvm/include/llvm/IR/Type.h +++ b/llvm/include/llvm/IR/Type.h @@ -236,6 +236,9 @@ return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID; } + /// True if this is an instance of scalable types. + bool isScalableType() const; + /// Return true if this type could be converted with a lossless BitCast to /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the /// same size only where no re-interpretation of the bits is done. diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -144,6 +144,22 @@ ScalarTy getFixed() const { return this->getValue(0); } ScalarTy getScalable() const { return this->getValue(1); } + + bool operator<(const StackOffset &RHS) const { + return (getFixed() < RHS.getFixed() && getScalable() < RHS.getScalable()) || + (getFixed() == RHS.getFixed() && + getScalable() < RHS.getScalable()) || + (getFixed() < RHS.getFixed() && getScalable() == RHS.getScalable()); + } + bool operator<=(const StackOffset &RHS) const { + return getFixed() <= RHS.getFixed() && getScalable() <= RHS.getScalable(); + } + bool operator>(const StackOffset &RHS) const { + return (getFixed() > RHS.getFixed() && getScalable() > RHS.getScalable()) || + (getFixed() == RHS.getFixed() && + getScalable() > RHS.getScalable()) || + (getFixed() > RHS.getFixed() && getScalable() == RHS.getScalable()); + } }; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -174,9 +174,12 @@ // Scalable vectors may need a special StackID to distinguish // them from other (fixed size) stack objects. - if (isa(Ty)) + if (Ty->isScalableType()) { + assert(Ty->isSized() && "It is only permitted to alloca struct " + "with the same scalable vector types."); MF->getFrameInfo().setStackID(FrameIndex, TFI->getStackIDForScalableVectors()); + } StaticAllocaMap[AI] = FrameIndex; // Update the catch handler information. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4009,7 +4009,6 @@ SmallVector Values(NumValues); SmallVector Chains(std::min(MaxParallelChains, NumValues)); - EVT PtrVT = Ptr.getValueType(); MachineMemOperand::Flags MMOFlags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); @@ -4029,10 +4028,12 @@ Root = Chain; ChainI = 0; } - SDValue A = DAG.getNode(ISD::ADD, dl, - PtrVT, Ptr, - DAG.getConstant(Offsets[i], dl, PtrVT), - Flags); + SDValue A; + if (ValueVTs[i].isScalableVector() && Offsets[i] != 0) + A = DAG.getMemBasePlusOffset(Ptr, TypeSize::Scalable(Offsets[i]), dl, + Flags); + else + A = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, MachinePointerInfo(SV, Offsets[i]), Alignment, @@ -4175,8 +4176,13 @@ Root = Chain; ChainI = 0; } - SDValue Add = - DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); + SDValue Add; + if (ValueVTs[i].isScalableVector() && Offsets[i] != 0) + Add = DAG.getMemBasePlusOffset(Ptr, TypeSize::Scalable(Offsets[i]), dl, + Flags); + else + Add = + DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); if (MemVTs[i] != ValueVTs[i]) Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp --- a/llvm/lib/IR/DataLayout.cpp +++ b/llvm/lib/IR/DataLayout.cpp @@ -44,11 +44,16 @@ // Support for StructLayout //===----------------------------------------------------------------------===// -StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { +StructLayout::StructLayout(StructType *ST, const DataLayout &DL) + : StructSize(0, false) { assert(!ST->isOpaque() && "Cannot get layout of opaque structs"); - StructSize = 0; + // We permit scalable vector struct as "sized" only when all the elements + // are the same scalable vector types. + if (ST->isScalableType()) + StructSize = TypeSize::Scalable(0); IsPadded = false; NumElements = ST->getNumElements(); + StackOffset Offset = StackOffset::getFixed(0); // Loop over each of the elements, placing them in memory. for (unsigned i = 0, e = NumElements; i != e; ++i) { @@ -56,37 +61,54 @@ const Align TyAlign = ST->isPacked() ? Align(1) : DL.getABITypeAlign(Ty); // Add padding if necessary to align the data element properly. - if (!isAligned(TyAlign, StructSize)) { + // Scalable vector struct must be the same scalable vector types. They have + // no alignment issues. + if (!StructSize.isScalable() && + !isAligned(TyAlign, StructSize.getFixedSize())) { IsPadded = true; - StructSize = alignTo(StructSize, TyAlign); + uint64_t Padding = alignTo(StructSize.getFixedSize(), TyAlign) - + StructSize.getFixedSize(); + Offset += StackOffset::getFixed(Padding); + StructSize = TypeSize::Fixed(alignTo(StructSize.getFixedSize(), TyAlign)); } // Keep track of maximum alignment constraint. StructAlignment = std::max(TyAlign, StructAlignment); - MemberOffsets[i] = StructSize; + MemberOffsets.push_back(Offset); // Consume space for this data item - StructSize += DL.getTypeAllocSize(Ty).getFixedValue(); + TypeSize ElemSize = DL.getTypeAllocSize(Ty); + StructSize += ElemSize; + if (Ty->isScalableType()) + Offset += StackOffset::getScalable(ElemSize.getKnownMinSize()); + else + Offset += StackOffset::getFixed(ElemSize.getFixedSize()); } // Add padding to the end of the struct so that it could be put in an array // and all array elements would be aligned correctly. - if (!isAligned(StructAlignment, StructSize)) { + if (!StructSize.isScalable() && + !isAligned(StructAlignment, StructSize.getFixedSize())) { IsPadded = true; - StructSize = alignTo(StructSize, StructAlignment); + StructSize = + TypeSize::Fixed(alignTo(StructSize.getFixedSize(), StructAlignment)); } } /// getElementContainingOffset - Given a valid offset into the structure, /// return the structure index that contains it. -unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const { - const uint64_t *SI = - std::upper_bound(&MemberOffsets[0], &MemberOffsets[NumElements], Offset); - assert(SI != &MemberOffsets[0] && "Offset not in structure type!"); +unsigned StructLayout::getElementContainingOffset(uint64_t O) const { + StackOffset Offset = StackOffset::getFixed(O); + if (StructSize.isScalable()) + Offset = StackOffset::getScalable(O); + const StackOffset *SI = std::upper_bound( + MemberOffsets.begin(), MemberOffsets.end(), Offset, + [](StackOffset lhs, StackOffset rhs) -> bool { return lhs < rhs; }); + assert(SI != MemberOffsets.begin() && "Offset not in structure type!"); --SI; assert(*SI <= Offset && "upper_bound didn't work"); - assert((SI == &MemberOffsets[0] || *(SI-1) <= Offset) && - (SI+1 == &MemberOffsets[NumElements] || *(SI+1) > Offset) && + assert((SI == MemberOffsets.begin() || *(SI - 1) <= Offset) && + (SI + 1 == MemberOffsets.end() || *(SI + 1) > Offset) && "Upper bound didn't work!"); // Multiple fields can have the same offset if any of them are zero sized. @@ -94,7 +116,7 @@ // at the i32 element, because it is the last element at that offset. This is // the right one to return, because anything after it will have a higher // offset, implying that this element is non-empty. - return SI-&MemberOffsets[0]; + return SI - MemberOffsets.begin(); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -173,6 +173,17 @@ return cast(this)->isSized(Visited); } +bool Type::isScalableType() const { + if (getTypeID() == ScalableVectorTyID) + return true; + + if (auto *STy = dyn_cast(this)) { + if (STy->containsScalableVectorType()) + return true; + } + return false; +} + //===----------------------------------------------------------------------===// // Primitive 'Type' data //===----------------------------------------------------------------------===// @@ -521,11 +532,22 @@ // Okay, our struct is sized if all of the elements are, but if one of the // elements is opaque, the struct isn't sized *yet*, but may become sized in // the future, so just bail out without caching. + Type *FirstTy = getNumElements() > 0 ? elements()[0] : nullptr; + bool IsFirstOneScalable = false; + if (FirstTy) + IsFirstOneScalable = isa(FirstTy); for (Type *Ty : elements()) { - // If the struct contains a scalable vector type, don't consider it sized. - // This prevents it from being used in loads/stores/allocas/GEPs. - if (isa(Ty)) - return false; + if (IsFirstOneScalable) { + // We do not permit mix scalar types with scalable types within struct. + if (!isa(Ty)) + return false; + // All the scalable types within struct should be the same. + if (FirstTy != Ty) + return false; + } else { + if (isa(Ty)) + return false; + } if (!Ty->isSized(Visited)) return false; } diff --git a/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll @@ -0,0 +1,47 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-v -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s + +target triple = "riscv64-unknown-unknown-elf" + +%struct.test = type { , } + +define @test(%struct.test* %addr, i64 %vl) { +; CHECK-LABEL: test: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: csrrs a2, vlenb, zero +; CHECK-NEXT: slli a2, a2, 1 +; CHECK-NEXT: sub sp, sp, a2 +; CHECK-NEXT: csrrs a2, vlenb, zero +; CHECK-NEXT: add a3, a0, a2 +; CHECK-NEXT: vl1re64.v v25, (a3) +; CHECK-NEXT: vl1re64.v v26, (a0) +; CHECK-NEXT: addi a0, sp, 0 +; CHECK-NEXT: add a0, a0, a2 +; CHECK-NEXT: vs1r.v v25, (a0) +; CHECK-NEXT: vs1r.v v26, (sp) +; CHECK-NEXT: vl1re64.v v25, (a0) +; CHECK-NEXT: vl1re64.v v26, (sp) +; CHECK-NEXT: vsetvli a0, a1, e64,m1,ta,mu +; CHECK-NEXT: vfadd.vv v8, v26, v25 +; CHECK-NEXT: csrrs a0, vlenb, zero +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: add sp, sp, a0 +; CHECK-NEXT: jalr zero, 0(ra) +entry: + %ret = alloca %struct.test, align 8 + %val = load %struct.test, %struct.test* %addr + store %struct.test %val, %struct.test* %ret, align 8 + %0 = load %struct.test, %struct.test* %ret, align 8 + %1 = extractvalue %struct.test %0, 0 + %2 = extractvalue %struct.test %0, 1 + %3 = call @llvm.riscv.vfadd.nxv1f64.nxv1f64( + %1, %2, i64 %vl) + ret %3 +} + +declare @llvm.riscv.vfadd.nxv1f64.nxv1f64( + , + , + i64); diff --git a/llvm/test/Other/load-scalable-vector-struct.ll b/llvm/test/Other/load-scalable-vector-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Other/load-scalable-vector-struct.ll @@ -0,0 +1,12 @@ +; RUN: opt -S -verify < %s 2>&1 | FileCheck %s + +%struct.test = type { , } + +define @load(%struct.test* %x) { +; CHECK: %a = load %struct.test, %struct.test* %x, align 4 +; CHECK: %b = extractvalue %struct.test %a, 1 +; CHECK: ret %b + %a = load %struct.test, %struct.test* %x + %b = extractvalue %struct.test %a, 1 + ret %b +} diff --git a/llvm/test/Other/store-scalable-vector-struct.ll b/llvm/test/Other/store-scalable-vector-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Other/store-scalable-vector-struct.ll @@ -0,0 +1,14 @@ +; RUN: opt -S -verify < %s 2>&1 | FileCheck %s + +%struct.test = type { , } + +define void @store(%struct.test* %x, %y, %z) { +; CHECK: %a = insertvalue %struct.test undef, %y, 0 +; CHECK: %b = insertvalue %struct.test %a, %z, 1 +; CHECK: store %struct.test %b, %struct.test* %x +; CHECK: ret void + %a = insertvalue %struct.test undef, %y, 0 + %b = insertvalue %struct.test %a, %z, 1 + store %struct.test %b, %struct.test* %x + ret void +}