diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h --- a/llvm/include/llvm/CodeGen/Analysis.h +++ b/llvm/include/llvm/CodeGen/Analysis.h @@ -67,15 +67,15 @@ /// void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl &ValueVTs, - SmallVectorImpl *Offsets = nullptr, - uint64_t StartingOffset = 0); + SmallVectorImpl *Offsets = nullptr, + TypeSize StartingOffset = TypeSize::Fixed(0)); /// Variant of ComputeValueVTs that also produces the memory VTs. void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl &ValueVTs, SmallVectorImpl *MemVTs, - SmallVectorImpl *Offsets = nullptr, - uint64_t StartingOffset = 0); + SmallVectorImpl *Offsets = nullptr, + TypeSize StartingOffset = TypeSize::Fixed(0)); /// computeValueLLTs - Given an LLVM IR type, compute a sequence of /// LLTs that represent all the individual underlying diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h --- a/llvm/include/llvm/IR/DataLayout.h +++ b/llvm/include/llvm/IR/DataLayout.h @@ -620,16 +620,16 @@ /// Used to lazily calculate structure layout information for a target machine, /// based on the DataLayout structure. class StructLayout { - uint64_t StructSize; + TypeSize StructSize; Align StructAlignment; unsigned IsPadded : 1; unsigned NumElements : 31; - uint64_t MemberOffsets[1]; // variable sized array! + SmallVector MemberOffsets; public: - uint64_t getSizeInBytes() const { return StructSize; } + TypeSize getSizeInBytes() const { return StructSize; } - uint64_t getSizeInBits() const { return 8 * StructSize; } + TypeSize getSizeInBits() const { return 8 * StructSize; } Align getAlignment() const { return StructAlignment; } @@ -641,12 +641,12 @@ /// index that contains it. unsigned getElementContainingOffset(uint64_t Offset) const; - uint64_t getElementOffset(unsigned Idx) const { + TypeSize getElementOffset(unsigned Idx) const { assert(Idx < NumElements && "Invalid element idx!"); return MemberOffsets[Idx]; } - uint64_t getElementOffsetInBits(unsigned Idx) const { + TypeSize getElementOffsetInBits(unsigned Idx) const { return getElementOffset(Idx) * 8; } @@ -672,8 +672,7 @@ } case Type::StructTyID: // Get the layout annotation... which is lazily created on demand. - return TypeSize::Fixed( - getStructLayout(cast(Ty))->getSizeInBits()); + return getStructLayout(cast(Ty))->getSizeInBits(); case Type::IntegerTyID: return TypeSize::Fixed(Ty->getIntegerBitWidth()); case Type::HalfTyID: diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h --- a/llvm/include/llvm/IR/Type.h +++ b/llvm/include/llvm/IR/Type.h @@ -236,6 +236,9 @@ return getTypeID() == ScalableVectorTyID || getTypeID() == FixedVectorTyID; } + /// True if this is an instance of scalable types. + bool isScalableType() const; + /// Return true if this type could be converted with a lossless BitCast to /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the /// same size only where no re-interpretation of the bits is done. diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -1003,6 +1003,7 @@ NewIdxs.push_back(ConstantInt::get(IntIdxTy, NewIdx)); } } else { + // TODO: Review for scalable struct. auto *STy = cast(Ty); // If we end up with an offset that isn't valid for this struct type, we // can't re-form this GEP in a regular form, so bail out. The pointer diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -212,7 +212,7 @@ const TargetLibraryInfo *TLI) { // For unsized types or scalable vectors we don't know exactly how many bytes // are dereferenced, so bail out. - if (!Ty->isSized() || isa(Ty)) + if (!Ty->isSized() || Ty->isScalableType()) return false; // When dereferenceability information is provided by a dereferenceable diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp --- a/llvm/lib/Analysis/MemoryBuiltins.cpp +++ b/llvm/lib/Analysis/MemoryBuiltins.cpp @@ -355,6 +355,7 @@ return nullptr; unsigned ElementSize = DL.getTypeAllocSize(T); + // TODO: Review for scalable struct. if (StructType *ST = dyn_cast(T)) ElementSize = DL.getStructLayout(ST)->getSizeInBytes(); diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3442,6 +3442,7 @@ SmallVector Offsets; for (const SCEV *IndexExpr : IndexExprs) { // Compute the (potentially symbolic) offset in bytes for this index. + // TODO: Review for scalable struct. if (StructType *STy = dyn_cast(CurTy)) { // For a struct, add the member offset. ConstantInt *Index = cast(IndexExpr)->getValue(); diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp --- a/llvm/lib/CodeGen/Analysis.cpp +++ b/llvm/lib/CodeGen/Analysis.cpp @@ -82,8 +82,8 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl &ValueVTs, SmallVectorImpl *MemVTs, - SmallVectorImpl *Offsets, - uint64_t StartingOffset) { + SmallVectorImpl *Offsets, + TypeSize StartingOffset) { // Given a struct type, recursively traverse the elements. if (StructType *STy = dyn_cast(Ty)) { // If the Offsets aren't needed, don't query the struct layout. This allows @@ -95,7 +95,8 @@ EE = STy->element_end(); EI != EE; ++EI) { // Don't compute the element offset if we didn't get a StructLayout above. - uint64_t EltOffset = SL ? SL->getElementOffset(EI - EB) : 0; + TypeSize EltOffset = + SL ? SL->getElementOffset(EI - EB) : TypeSize::Fixed(0); ComputeValueVTs(TLI, DL, *EI, ValueVTs, MemVTs, Offsets, StartingOffset + EltOffset); } @@ -104,10 +105,10 @@ // Given an array type, recursively traverse the elements. if (ArrayType *ATy = dyn_cast(Ty)) { Type *EltTy = ATy->getElementType(); - uint64_t EltSize = DL.getTypeAllocSize(EltTy).getFixedValue(); + TypeSize EltSize = DL.getTypeAllocSize(EltTy); for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) ComputeValueVTs(TLI, DL, EltTy, ValueVTs, MemVTs, Offsets, - StartingOffset + i * EltSize); + StartingOffset + EltSize * i); return; } // Interpret void as zero return values. @@ -123,8 +124,8 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl &ValueVTs, - SmallVectorImpl *Offsets, - uint64_t StartingOffset) { + SmallVectorImpl *Offsets, + TypeSize StartingOffset) { return ComputeValueVTs(TLI, DL, Ty, ValueVTs, /*MemVTs=*/nullptr, Offsets, StartingOffset); } @@ -134,6 +135,7 @@ SmallVectorImpl *Offsets, uint64_t StartingOffset) { // Given a struct type, recursively traverse the elements. + // TODO: Review for scalable struct. if (StructType *STy = dyn_cast(&Ty)) { // If the Offsets aren't needed, don't query the struct layout. This allows // us to support structs with scalable vectors for operations that don't diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -7431,7 +7431,7 @@ // whereas scalable vectors would have to be shifted by // <2log(vscale) + number of bits> in order to store the // low/high parts. Bailing out for now. - if (isa(StoreType)) + if (StoreType->isScalableType()) return false; if (!DL.typeSizeEqualsStoreSize(StoreType) || diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -194,8 +194,8 @@ LLVMContext &Ctx = OrigArg.Ty->getContext(); SmallVector SplitVTs; - SmallVector Offsets; - ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); + SmallVector Offsets; + ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, TypeSize::Fixed(0)); if (SplitVTs.size() == 0) return; @@ -724,8 +724,8 @@ const DataLayout &DL = MF.getDataLayout(); SmallVector SplitVTs; - SmallVector Offsets; - ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); + SmallVector Offsets; + ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, TypeSize::Fixed(0)); assert(VRegs.size() == SplitVTs.size()); @@ -754,8 +754,8 @@ const DataLayout &DL = MF.getDataLayout(); SmallVector SplitVTs; - SmallVector Offsets; - ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0); + SmallVector Offsets; + ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, TypeSize::Fixed(0)); assert(VRegs.size() == SplitVTs.size()); diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -174,9 +174,12 @@ // Scalable vectors may need a special StackID to distinguish // them from other (fixed size) stack objects. - if (isa(Ty)) + if (Ty->isScalableType()) { + assert(Ty->isSized() && "It is only permitted to alloca struct " + "with the same scalable vector types."); MF->getFrameInfo().setStackID(FrameIndex, TFI->getStackIDForScalableVectors()); + } StaticAllocaMap[AI] = FrameIndex; // Update the catch handler information. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1887,7 +1887,7 @@ SDValue RetOp = getValue(I.getOperand(0)); SmallVector ValueVTs, MemVTs; - SmallVector Offsets; + SmallVector Offsets; ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs, &Offsets); unsigned NumValues = ValueVTs.size(); @@ -1897,8 +1897,7 @@ for (unsigned i = 0; i != NumValues; ++i) { // An aggregate return value cannot wrap around the address space, so // offsets to its parts don't wrap either. - SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, - TypeSize::Fixed(Offsets[i])); + SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]); SDValue Val = RetOp.getValue(RetOp.getResNo() + i); if (MemVTs[i] != ValueVTs[i]) @@ -4039,8 +4038,12 @@ const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); SmallVector ValueVTs, MemVTs; - SmallVector Offsets; - ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); + SmallVector Offsets; + TypeSize StartingOffset(0, false); + if (Ty->isScalableType()) + StartingOffset = TypeSize::Scalable(0); + ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets, + StartingOffset); unsigned NumValues = ValueVTs.size(); if (NumValues == 0) return; @@ -4079,7 +4082,6 @@ SmallVector Values(NumValues); SmallVector Chains(std::min(MaxParallelChains, NumValues)); - EVT PtrVT = Ptr.getValueType(); MachineMemOperand::Flags MMOFlags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); @@ -4099,14 +4101,12 @@ Root = Chain; ChainI = 0; } - SDValue A = DAG.getNode(ISD::ADD, dl, - PtrVT, Ptr, - DAG.getConstant(Offsets[i], dl, PtrVT), - Flags); + SDValue A = DAG.getMemBasePlusOffset(Ptr, Offsets[i], dl, Flags); - SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, - MachinePointerInfo(SV, Offsets[i]), Alignment, - MMOFlags, AAInfo, Ranges); + SDValue L = + DAG.getLoad(MemVTs[i], dl, Root, A, + MachinePointerInfo(SV, Offsets[i].getKnownMinSize()), + Alignment, MMOFlags, AAInfo, Ranges); Chains[ChainI] = L.getValue(1); if (MemVTs[i] != ValueVTs[i]) @@ -4133,10 +4133,13 @@ "call visitStoreToSwiftError when backend supports swifterror"); SmallVector ValueVTs; - SmallVector Offsets; + SmallVector Offsets; const Value *SrcV = I.getOperand(0); + TypeSize StartingOffset(0, false); + if (SrcV->getType()->isScalableType()) + StartingOffset = TypeSize::Scalable(0); ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), - SrcV->getType(), ValueVTs, &Offsets); + SrcV->getType(), ValueVTs, &Offsets, StartingOffset); assert(ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"); @@ -4172,9 +4175,12 @@ "load_from_swift_error should not be constant memory"); SmallVector ValueVTs; - SmallVector Offsets; + SmallVector Offsets; + TypeSize StartingOffset(0, false); + if (Ty->isScalableType()) + StartingOffset = TypeSize::Scalable(0); ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty, - ValueVTs, &Offsets); + ValueVTs, &Offsets, StartingOffset); assert(ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"); @@ -4209,9 +4215,12 @@ } SmallVector ValueVTs, MemVTs; - SmallVector Offsets; + SmallVector Offsets; + TypeSize StartingOffset(0, false); + if (SrcV->getType()->isScalableType()) + StartingOffset = TypeSize::Scalable(0); ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), - SrcV->getType(), ValueVTs, &MemVTs, &Offsets); + SrcV->getType(), ValueVTs, &MemVTs, &Offsets, StartingOffset); unsigned NumValues = ValueVTs.size(); if (NumValues == 0) return; @@ -4245,13 +4254,13 @@ Root = Chain; ChainI = 0; } - SDValue Add = - DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); + SDValue Add = DAG.getMemBasePlusOffset(Ptr, Offsets[i], dl, Flags); SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); if (MemVTs[i] != ValueVTs[i]) Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); SDValue St = - DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), + DAG.getStore(Root, dl, Val, Add, + MachinePointerInfo(PtrV, Offsets[i].getKnownMinSize()), Alignment, MMOFlags, AAInfo); Chains[ChainI] = St; } @@ -9281,26 +9290,29 @@ CLI.Ins.clear(); Type *OrigRetTy = CLI.RetTy; SmallVector RetTys; - SmallVector Offsets; + SmallVector Offsets; auto &DL = CLI.DAG.getDataLayout(); - ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); + TypeSize StartingOffset(0, false); + if (CLI.RetTy->isScalableType()) + StartingOffset = TypeSize::Scalable(0); + ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets, StartingOffset); if (CLI.IsPostTypeLegalization) { // If we are lowering a libcall after legalization, split the return type. SmallVector OldRetTys; - SmallVector OldOffsets; + SmallVector OldOffsets; RetTys.swap(OldRetTys); Offsets.swap(OldOffsets); for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) { EVT RetVT = OldRetTys[i]; - uint64_t Offset = OldOffsets[i]; + TypeSize Offset = OldOffsets[i]; MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT); unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT); unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; RetTys.append(NumRegs, RegisterVT); for (unsigned j = 0; j != NumRegs; ++j) - Offsets.push_back(Offset + j * RegisterVTByteSZ); + Offsets.push_back(Offset + TypeSize::Fixed(j * RegisterVTByteSZ)); } } @@ -9602,7 +9614,6 @@ ComputeValueVTs(*this, DL, PtrRetTy, PVTs); assert(PVTs.size() == 1 && "Pointers should fit in one register"); - EVT PtrVT = PVTs[0]; unsigned NumValues = RetTys.size(); ReturnValues.resize(NumValues); @@ -9616,9 +9627,9 @@ MachineFunction &MF = CLI.DAG.getMachineFunction(); Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx); for (unsigned i = 0; i < NumValues; ++i) { - SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, - CLI.DAG.getConstant(Offsets[i], CLI.DL, - PtrVT), Flags); + SDValue Add = CLI.DAG.getMemBasePlusOffset(DemoteStackSlot, Offsets[i], + CLI.DL, Flags); + SDValue L = CLI.DAG.getLoad( RetTys[i], CLI.DL, CLI.Chain, Add, MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(), diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp --- a/llvm/lib/IR/DataLayout.cpp +++ b/llvm/lib/IR/DataLayout.cpp @@ -44,11 +44,16 @@ // Support for StructLayout //===----------------------------------------------------------------------===// -StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { +StructLayout::StructLayout(StructType *ST, const DataLayout &DL) + : StructSize(0, false) { assert(!ST->isOpaque() && "Cannot get layout of opaque structs"); - StructSize = 0; + // We permit scalable vector struct as "sized" only when all the elements + // are the same scalable vector types. + if (ST->isScalableType()) + StructSize = TypeSize::Scalable(0); IsPadded = false; NumElements = ST->getNumElements(); + TypeSize Offset(0, StructSize.isScalable()); // Loop over each of the elements, placing them in memory. for (unsigned i = 0, e = NumElements; i != e; ++i) { @@ -56,45 +61,61 @@ const Align TyAlign = ST->isPacked() ? Align(1) : DL.getABITypeAlign(Ty); // Add padding if necessary to align the data element properly. - if (!isAligned(TyAlign, StructSize)) { + // Scalable vector struct must be the same scalable vector types. They have + // no alignment issues. + if (!StructSize.isScalable() && + !isAligned(TyAlign, StructSize.getFixedSize())) { IsPadded = true; - StructSize = alignTo(StructSize, TyAlign); + uint64_t Padding = alignTo(StructSize.getFixedSize(), TyAlign) - + StructSize.getFixedSize(); + Offset += TypeSize::Fixed(Padding); + StructSize = TypeSize::Fixed(alignTo(StructSize.getFixedSize(), TyAlign)); } // Keep track of maximum alignment constraint. StructAlignment = std::max(TyAlign, StructAlignment); - MemberOffsets[i] = StructSize; + MemberOffsets.push_back(Offset); // Consume space for this data item - StructSize += DL.getTypeAllocSize(Ty).getFixedValue(); + TypeSize ElemSize = DL.getTypeAllocSize(Ty); + StructSize += ElemSize; + Offset += ElemSize; } // Add padding to the end of the struct so that it could be put in an array // and all array elements would be aligned correctly. - if (!isAligned(StructAlignment, StructSize)) { + if (!StructSize.isScalable() && + !isAligned(StructAlignment, StructSize.getFixedSize())) { IsPadded = true; - StructSize = alignTo(StructSize, StructAlignment); + StructSize = + TypeSize::Fixed(alignTo(StructSize.getFixedSize(), StructAlignment)); } } /// getElementContainingOffset - Given a valid offset into the structure, /// return the structure index that contains it. -unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const { - const uint64_t *SI = - std::upper_bound(&MemberOffsets[0], &MemberOffsets[NumElements], Offset); - assert(SI != &MemberOffsets[0] && "Offset not in structure type!"); +unsigned StructLayout::getElementContainingOffset(uint64_t O) const { + TypeSize Offset(O, StructSize.isScalable()); + const TypeSize *SI = + std::upper_bound(MemberOffsets.begin(), MemberOffsets.end(), Offset, + [](TypeSize lhs, TypeSize rhs) -> bool { + return TypeSize::isKnownLT(lhs, rhs); + }); + assert(SI != MemberOffsets.begin() && "Offset not in structure type!"); --SI; - assert(*SI <= Offset && "upper_bound didn't work"); - assert((SI == &MemberOffsets[0] || *(SI-1) <= Offset) && - (SI+1 == &MemberOffsets[NumElements] || *(SI+1) > Offset) && - "Upper bound didn't work!"); + assert(TypeSize::isKnownLE(*SI, Offset) && "upper_bound didn't work"); + assert( + (SI == MemberOffsets.begin() || TypeSize::isKnownLE(*(SI - 1), Offset)) && + (SI + 1 == MemberOffsets.end() || + TypeSize::isKnownLT(Offset, *(SI + 1))) && + "Upper bound didn't work!"); // Multiple fields can have the same offset if any of them are zero sized. // For example, in { i32, [0 x i32], i32 }, searching for offset 4 will stop // at the i32 element, because it is the last element at that offset. This is // the right one to return, because anything after it will have a higher // offset, implying that this element is non-empty. - return SI-&MemberOffsets[0]; + return SI - MemberOffsets.begin(); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -180,6 +180,17 @@ return cast(this)->isSized(Visited); } +bool Type::isScalableType() const { + if (getTypeID() == ScalableVectorTyID) + return true; + + if (auto *STy = dyn_cast(this)) { + if (STy->containsScalableVectorType()) + return true; + } + return false; +} + //===----------------------------------------------------------------------===// // Primitive 'Type' data //===----------------------------------------------------------------------===// @@ -528,11 +539,22 @@ // Okay, our struct is sized if all of the elements are, but if one of the // elements is opaque, the struct isn't sized *yet*, but may become sized in // the future, so just bail out without caching. + Type *FirstTy = getNumElements() > 0 ? elements()[0] : nullptr; + bool IsFirstOneScalable = false; + if (FirstTy) + IsFirstOneScalable = isa(FirstTy); for (Type *Ty : elements()) { - // If the struct contains a scalable vector type, don't consider it sized. - // This prevents it from being used in loads/stores/allocas/GEPs. - if (isa(Ty)) - return false; + if (IsFirstOneScalable) { + // We do not permit mix scalar types with scalable types within struct. + if (!isa(Ty)) + return false; + // All the scalable types within struct should be the same. + if (FirstTy != Ty) + return false; + } else { + if (isa(Ty)) + return false; + } if (!Ty->isSized(Visited)) return false; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -1042,8 +1042,9 @@ // the correct memory offsets. SmallVector ValueVTs; - SmallVector Offsets; - ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, &Offsets, ArgOffset); + SmallVector Offsets; + ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, &Offsets, + TypeSize::Fixed(ArgOffset)); for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; ++Value) { diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -150,7 +150,7 @@ SmallVectorImpl *Offsets = nullptr, uint64_t StartingOffset = 0) { SmallVector TempVTs; - SmallVector TempOffsets; + SmallVector TempOffsets; // Special case for i128 - decompose to (i64, i64) if (Ty->isIntegerTy(128)) { @@ -177,7 +177,8 @@ return; } - ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset); + ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, + TypeSize::Fixed(StartingOffset)); for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) { EVT VT = TempVTs[i]; uint64_t Off = TempOffsets[i]; diff --git a/llvm/lib/Target/X86/X86CallLowering.cpp b/llvm/lib/Target/X86/X86CallLowering.cpp --- a/llvm/lib/Target/X86/X86CallLowering.cpp +++ b/llvm/lib/Target/X86/X86CallLowering.cpp @@ -60,8 +60,8 @@ LLVMContext &Context = OrigArg.Ty->getContext(); SmallVector SplitVTs; - SmallVector Offsets; - ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); + SmallVector Offsets; + ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, TypeSize::Fixed(0)); assert(OrigArg.Regs.size() == 1 && "Can't handle multple regs yet"); if (OrigArg.Ty->isVoidTy()) diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp --- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -797,6 +797,7 @@ return true; // Check for padding within and between elements of a struct. + // TODO: Review for scalable struct. StructType *StructTy = cast(type); const StructLayout *Layout = DL.getStructLayout(StructTy); uint64_t StartPos = 0; diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -213,6 +213,7 @@ // Non-aggregate type, we cast and make byte-wise progress now. break; + // TODO: Review for scalable struct. const StructLayout *SL = DL.getStructLayout(STy); if (int64_t(SL->getSizeInBytes()) < Offset) break; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -628,6 +628,7 @@ StringRef Name = LI.getName(); assert(LI.getAlignment() && "Alignment must be set at this point"); + // TODO: Review for scalable struct. if (auto *ST = dyn_cast(T)) { // If the struct only have one element, we unpack. auto NumElements = ST->getNumElements(); @@ -1157,6 +1158,7 @@ if (!T->isAggregateType()) return false; + // TODO: Review for scalable struct. if (auto *ST = dyn_cast(T)) { // If the struct only have one element, we unpack. unsigned Count = ST->getNumElements(); diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1260,6 +1260,7 @@ return nullptr; if (StructType *STy = dyn_cast(Ty)) { + // TODO: Review for scalable struct. const StructLayout *SL = DL.getStructLayout(STy); assert(Offset < (int64_t)SL->getSizeInBytes() && "Offset must stay within the indexed type"); diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -1506,6 +1506,7 @@ Indices, NamePrefix); } + // TODO: Review for scalable struct. StructType *STy = dyn_cast(Ty); if (!STy) return nullptr; @@ -3620,6 +3621,7 @@ if (ArrayType *ArrTy = dyn_cast(Ty)) { InnerTy = ArrTy->getElementType(); } else if (StructType *STy = dyn_cast(Ty)) { + // TODO: Review for scalable struct. const StructLayout *SL = DL.getStructLayout(STy); unsigned Index = SL->getElementContainingOffset(0); InnerTy = STy->getElementType(Index); @@ -3693,6 +3695,7 @@ return ArrayType::get(ElementTy, NumElements); } + // TODO: Review for scalable struct. StructType *STy = dyn_cast(Ty); if (!STy) return nullptr; diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -498,6 +498,7 @@ break; if (const SCEVConstant *C = dyn_cast(Ops[0])) if (SE.getTypeSizeInBits(C->getType()) <= 64) { + // TODO: Review for scalable struct. const StructLayout &SL = *DL.getStructLayout(STy); uint64_t FullOffset = C->getValue()->getZExtValue(); if (FullOffset < SL.getSizeInBytes()) { diff --git a/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/load-store-scalable-struct.ll @@ -0,0 +1,51 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+experimental-v -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s + +target triple = "riscv64-unknown-unknown-elf" + +%struct.test = type { , } + +define @test(%struct.test* %addr, i64 %vl) { +; CHECK-LABEL: test: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: addi sp, sp, -16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: csrrs a2, vlenb, zero +; CHECK-NEXT: slli a2, a2, 1 +; CHECK-NEXT: sub sp, sp, a2 +; CHECK-NEXT: csrrs a2, vlenb, zero +; CHECK-NEXT: add a3, a0, a2 +; CHECK-NEXT: vl1re64.v v25, (a3) +; CHECK-NEXT: vl1re64.v v26, (a0) +; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: add a0, a0, a2 +; CHECK-NEXT: vs1r.v v25, (a0) +; CHECK-NEXT: addi a2, sp, 16 +; CHECK-NEXT: vs1r.v v26, (a2) +; CHECK-NEXT: vl1re64.v v25, (a0) +; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vl1re64.v v26, (a0) +; CHECK-NEXT: vsetvli a0, a1, e64,m1,ta,mu +; CHECK-NEXT: vfadd.vv v8, v26, v25 +; CHECK-NEXT: csrrs a0, vlenb, zero +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: add sp, sp, a0 +; CHECK-NEXT: addi sp, sp, 16 +; CHECK-NEXT: jalr zero, 0(ra) +entry: + %ret = alloca %struct.test, align 8 + %val = load %struct.test, %struct.test* %addr + store %struct.test %val, %struct.test* %ret, align 8 + %0 = load %struct.test, %struct.test* %ret, align 8 + %1 = extractvalue %struct.test %0, 0 + %2 = extractvalue %struct.test %0, 1 + %3 = call @llvm.riscv.vfadd.nxv1f64.nxv1f64( + %1, %2, i64 %vl) + ret %3 +} + +declare @llvm.riscv.vfadd.nxv1f64.nxv1f64( + , + , + i64); diff --git a/llvm/test/Other/load-scalable-vector-struct.ll b/llvm/test/Other/load-scalable-vector-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Other/load-scalable-vector-struct.ll @@ -0,0 +1,12 @@ +; RUN: opt -S -verify < %s 2>&1 | FileCheck %s + +%struct.test = type { , } + +define @load(%struct.test* %x) { +; CHECK: %a = load %struct.test, %struct.test* %x, align 4 +; CHECK: %b = extractvalue %struct.test %a, 1 +; CHECK: ret %b + %a = load %struct.test, %struct.test* %x + %b = extractvalue %struct.test %a, 1 + ret %b +} diff --git a/llvm/test/Other/store-scalable-vector-struct.ll b/llvm/test/Other/store-scalable-vector-struct.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Other/store-scalable-vector-struct.ll @@ -0,0 +1,14 @@ +; RUN: opt -S -verify < %s 2>&1 | FileCheck %s + +%struct.test = type { , } + +define void @store(%struct.test* %x, %y, %z) { +; CHECK: %a = insertvalue %struct.test undef, %y, 0 +; CHECK: %b = insertvalue %struct.test %a, %z, 1 +; CHECK: store %struct.test %b, %struct.test* %x +; CHECK: ret void + %a = insertvalue %struct.test undef, %y, 0 + %b = insertvalue %struct.test %a, %z, 1 + store %struct.test %b, %struct.test* %x + ret void +}