Index: llvm/include/llvm/Support/TypeSize.h =================================================================== --- llvm/include/llvm/Support/TypeSize.h +++ llvm/include/llvm/Support/TypeSize.h @@ -49,6 +49,13 @@ return { Min / RHS, Scalable }; } + friend ElementCount operator-(const ElementCount &LHS, + const ElementCount &RHS) { + assert(LHS.Scalable == RHS.Scalable && + "Arithmetic using mixed scalable and fixed types"); + return {LHS.Min - RHS.Min, LHS.Scalable}; + } + bool operator==(const ElementCount& RHS) const { return Min == RHS.Min && Scalable == RHS.Scalable; } @@ -219,6 +226,8 @@ return IsScalable; } + bool isPowerOf2() const { return (MinSize & (MinSize - 1)) == 0; } + // Returns true if the number of bits is a multiple of an 8-bit byte. bool isByteSized() const { return (MinSize & 7) == 0; Index: llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -718,7 +718,7 @@ LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n"); EVT SrcVT = LD->getMemoryVT(); - unsigned SrcWidth = SrcVT.getSizeInBits(); + TypeSize SrcWidth = SrcVT.getSizeInBits(); MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); AAMDNodes AAInfo = LD->getAAInfo(); @@ -764,14 +764,15 @@ Value = Result; Chain = Ch; - } else if (SrcWidth & (SrcWidth - 1)) { + } else if (!SrcWidth.isPowerOf2()) { // If not loading a power-of-2 number of bits, expand as two loads. assert(!SrcVT.isVector() && "Unsupported extload!"); - unsigned LogSrcWidth = Log2_32(SrcWidth); + unsigned SrcWidthBits = SrcWidth.getFixedSize(); + unsigned LogSrcWidth = Log2_32(SrcWidthBits); assert(LogSrcWidth < 32); unsigned RoundWidth = 1 << LogSrcWidth; - assert(RoundWidth < SrcWidth); - unsigned ExtraWidth = SrcWidth - RoundWidth; + assert(RoundWidth < SrcWidthBits); + unsigned ExtraWidth = SrcWidthBits - RoundWidth; assert(ExtraWidth < RoundWidth); assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"); Index: llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1636,9 +1636,10 @@ else std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); + unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize()); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( - MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), - Alignment, MLD->getAAInfo(), MLD->getRanges()); + MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment, + MLD->getAAInfo(), MLD->getRanges()); Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT, MMO, MLD->getAddressingMode(), ExtType, @@ -1652,12 +1653,18 @@ // Generate hi masked load. Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, MLD->isExpandingLoad()); - unsigned HiOffset = LoMemVT.getStoreSize(); + unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize()); + + MachinePointerInfo MPI; + if (LoMemVT.isScalableVector()) + MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace()); + else + MPI = MLD->getPointerInfo().getWithOffset( + LoMemVT.getStoreSize().getFixedSize()); MMO = DAG.getMachineFunction().getMachineMemOperand( - MLD->getPointerInfo().getWithOffset(HiOffset), - MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), Alignment, - MLD->getAAInfo(), MLD->getRanges()); + MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(), + MLD->getRanges()); Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, HiMemVT, MMO, MLD->getAddressingMode(), ExtType, Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6979,7 +6979,7 @@ assert(VT.isVector() == MemVT.isVector() && "Cannot use an ext load to convert to or from a vector!"); assert((!VT.isVector() || - VT.getVectorNumElements() == MemVT.getVectorNumElements()) && + VT.getVectorElementCount() == MemVT.getVectorElementCount()) && "Cannot use an ext load to change the number of vector elements!"); } @@ -9633,24 +9633,24 @@ SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const { EVT EltTp = VT.getVectorElementType(); - bool IsScalable = VT.isScalableVector(); // Examples: // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) // custom VL=9 with enveloping VL=8/8 yields 8/1 // custom VL=10 with enveloping VL=8/8 yields 8/2 // etc. - unsigned VTNumElts = VT.getVectorNumElements(); - unsigned EnvNumElts = EnvVT.getVectorNumElements(); + ElementCount VTNumElts = VT.getVectorElementCount(); + ElementCount EnvNumElts = EnvVT.getVectorElementCount(); + assert(VTNumElts.Scalable == EnvNumElts.Scalable && + "Mixing fixed width and scalable vectors when enveloping a type"); EVT LoVT, HiVT; - if (VTNumElts > EnvNumElts) { + if (VTNumElts.Min > EnvNumElts.Min) { LoVT = EnvVT; - HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts, - IsScalable); + HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); *HiIsEmpty = false; } else { // Flag that hi type has zero storage size, but return split envelop type // (this would be easier if vector types with zero elements were allowed). - LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts, IsScalable); + LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); HiVT = EnvVT; *HiIsEmpty = true; } Index: llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -7205,7 +7205,7 @@ SDValue Increment; EVT AddrVT = Addr.getValueType(); EVT MaskVT = Mask.getValueType(); - assert(DataVT.getVectorNumElements() == MaskVT.getVectorNumElements() && + assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && "Incompatible types of Data and Mask"); if (IsCompressedMemory) { if (DataVT.isScalableVector()) Index: llvm/test/CodeGen/AArch64/sve-split-load.ll =================================================================== --- llvm/test/CodeGen/AArch64/sve-split-load.ll +++ llvm/test/CodeGen/AArch64/sve-split-load.ll @@ -1,5 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning ; UNPREDICATED