Index: include/llvm/IR/DataLayout.h =================================================================== --- include/llvm/IR/DataLayout.h +++ include/llvm/IR/DataLayout.h @@ -119,6 +119,9 @@ /// Defaults to false. bool BigEndian; + /// Smallest number of bits in an addressable unit. Defaults to 8. + unsigned BitsPerAddressableUnit; + unsigned AllocaAddrSpace; unsigned StackNaturalAlign; unsigned ProgramAddrSpace; @@ -212,6 +215,7 @@ FunctionPtrAlign = DL.FunctionPtrAlign; TheFunctionPtrAlignType = DL.TheFunctionPtrAlignType; ProgramAddrSpace = DL.ProgramAddrSpace; + BitsPerAddressableUnit = DL.BitsPerAddressableUnit; ManglingMode = DL.ManglingMode; LegalIntWidths = DL.LegalIntWidths; Alignments = DL.Alignments; @@ -232,6 +236,9 @@ bool isLittleEndian() const { return !BigEndian; } bool isBigEndian() const { return BigEndian; } + /// Returns the smallest number of bits per addressable unit for the target. + unsigned getBitsPerAU() const { return BitsPerAddressableUnit; } + /// Returns the string representation of the DataLayout. /// /// This representation is in the same format accepted by the string @@ -411,7 +418,7 @@ unsigned getIndexTypeSizeInBits(Type *Ty) const; unsigned getPointerTypeSize(Type *Ty) const { - return getPointerTypeSizeInBits(Ty) / 8; + return getPointerTypeSizeInBits(Ty) / BitsPerAddressableUnit; } /// Size examples: Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -211,6 +211,16 @@ PruningList.insert(N); } + /// Return number of bits per addressable unit. + LLVM_READONLY unsigned bitsPerAU() const { + return DAG.getDataLayout().getBitsPerAU(); + } + + /// Return minimum number of addressable units needed to store \p Bits. + LLVM_READONLY unsigned inAUs(unsigned Bits) const { + return (Bits + bitsPerAU() - 1) / bitsPerAU(); + } + /// Add to the worklist making sure its instance is at the back (next to be /// processed.) void AddToWorklist(SDNode *N) { @@ -4463,7 +4473,7 @@ if (!LDST) return false; // Only allow byte offsets. - if (ShAmt % 8) + if (ShAmt % bitsPerAU()) return false; // Do not generate loads of non-round integer types since these can @@ -4482,7 +4492,7 @@ // Ensure that this isn't going to produce an unsupported unaligned access. if (ShAmt && !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT, - LDST->getAddressSpace(), ShAmt / 8)) + LDST->getAddressSpace(), ShAmt / bitsPerAU())) return false; // It's not possible to generate a constant of extended or untyped type. @@ -9771,7 +9781,7 @@ ShAmt = AdjustBigEndianShift(ShAmt); EVT PtrType = N0.getOperand(1).getValueType(); - uint64_t PtrOff = ShAmt / 8; + uint64_t PtrOff = ShAmt / bitsPerAU(); unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff); SDLoc DL(LN0); // The original load itself didn't wrap, so an offset within it doesn't. @@ -13429,12 +13439,12 @@ // n:th least significant byte of the stored value. if (DAG.getDataLayout().isBigEndian()) Offset = (STMemType.getStoreSizeInBits() - - LDMemType.getStoreSizeInBits()) / 8 - Offset; + LDMemType.getStoreSizeInBits()) / bitsPerAU() - Offset; // Check that the stored value cover all bits that are loaded. bool STCoversLD = (Offset >= 0) && - (Offset * 8 + LDMemType.getSizeInBits() <= STMemType.getSizeInBits()); + (Offset * bitsPerAU() + LDMemType.getSizeInBits() <= STMemType.getSizeInBits()); auto ReplaceLd = [&](LoadSDNode *LD, SDValue Val, SDValue Chain) -> SDValue { if (LD->isIndexed()) { @@ -13736,6 +13746,12 @@ unsigned Shift = 0, SelectionDAG *DAG = nullptr) : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {} + /// Return number of bits per addressable unit. + LLVM_READONLY unsigned bitsPerAU() const { + assert(DAG && "SelectionDAG not set."); + return DAG->getDataLayout().getBitsPerAU(); + } + /// Get the bits used in a chunk of bits \p BitWidth large. /// \return Result is \p BitWidth and has used bits set to 1 and /// not used bits set to 0. @@ -13759,8 +13775,9 @@ /// Get the size of the slice to be loaded in bytes. unsigned getLoadedSize() const { unsigned SliceSize = getUsedBits().countPopulation(); - assert(!(SliceSize & 0x7) && "Size is not a multiple of a byte."); - return SliceSize / 8; + assert(SliceSize % bitsPerAU() == 0 && + "Size is not a multiple of an addressable unit."); + return SliceSize / bitsPerAU(); } /// Get the type that will be loaded for this slice. @@ -13768,7 +13785,7 @@ EVT getLoadedType() const { assert(DAG && "Missing context"); LLVMContext &Ctxt = *DAG->getContext(); - return EVT::getIntegerVT(Ctxt, getLoadedSize() * 8); + return EVT::getIntegerVT(Ctxt, getLoadedSize() * bitsPerAU()); } /// Get the alignment of the load used for this slice. @@ -13829,13 +13846,16 @@ /// \pre DAG != nullptr. uint64_t getOffsetFromBase() const { assert(DAG && "Missing context."); - bool IsBigEndian = DAG->getDataLayout().isBigEndian(); - assert(!(Shift & 0x7) && "Shifts not aligned on Bytes are not supported."); - uint64_t Offset = Shift / 8; - unsigned TySizeInBytes = Origin->getValueSizeInBits(0) / 8; - assert(!(Origin->getValueSizeInBits(0) & 0x7) && - "The size of the original loaded type is not a multiple of a" - " byte."); + const DataLayout &TD = DAG->getDataLayout(); + bool IsBigEndian = TD.isBigEndian(); + unsigned BitsPerAU = TD.getBitsPerAU(); + assert(Shift % BitsPerAU == 0 && + "Shifts not aligned on addressable units are not supported."); + uint64_t Offset = Shift / BitsPerAU; + unsigned TySizeInBytes = Origin->getValueSizeInBits(0) / BitsPerAU; + assert(Origin->getValueSizeInBits(0) % BitsPerAU == 0 && + "The size of the original loaded type is not a multiple of an" + " addressable unit."); // If Offset is bigger than TySizeInBytes, it means we are loading all // zeros. This should have been optimized before in the process. assert(TySizeInBytes > Offset && @@ -14113,10 +14133,10 @@ // The width of the type must be a power of 2 and greater than 8-bits. // Otherwise the load cannot be represented in LLVM IR. - // Moreover, if we shifted with a non-8-bits multiple, the slice + // Moreover, if we shifted with a non-bitsPerAU multiple, the slice // will be across several bytes. We do not support that. unsigned Width = User->getValueSizeInBits(0); - if (Width < 8 || !isPowerOf2_32(Width) || (Shift & 0x7)) + if (Width < 8 || !isPowerOf2_32(Width) || (Shift % bitsPerAU())) return false; // Build the slice for this chain of computations. @@ -14254,17 +14274,19 @@ unsigned NumBytes = MaskInfo.first; unsigned ByteShift = MaskInfo.second; SelectionDAG &DAG = DC->getDAG(); + unsigned BitsPerAU = DC->bitsPerAU(); // Check to see if IVal is all zeros in the part being masked in by the 'or' // that uses this. If not, this is not a replacement. APInt Mask = ~APInt::getBitsSet(IVal.getValueSizeInBits(), - ByteShift*8, (ByteShift+NumBytes)*8); + ByteShift * BitsPerAU, + (ByteShift + NumBytes) * BitsPerAU); if (!DAG.MaskedValueIsZero(IVal, Mask)) return nullptr; // Check that it is legal on the target to do this. It is legal if the new // VT we're shrinking to (i8/i16/i32) is legal or we're still before type // legalization. - MVT VT = MVT::getIntegerVT(NumBytes*8); + MVT VT = MVT::getIntegerVT(NumBytes*BitsPerAU); if (!DC->isTypeLegal(VT)) return nullptr; @@ -14273,7 +14295,7 @@ if (ByteShift) { SDLoc DL(IVal); IVal = DAG.getNode(ISD::SRL, DL, IVal.getValueType(), IVal, - DAG.getConstant(ByteShift*8, DL, + DAG.getConstant(ByteShift * BitsPerAU, DL, DC->getShiftAmountTy(IVal.getValueType()))); } @@ -14391,11 +14413,11 @@ APInt NewImm = (Imm & Mask).lshr(ShAmt).trunc(NewBW); if (Opc == ISD::AND) NewImm ^= APInt::getAllOnesValue(NewBW); - uint64_t PtrOff = ShAmt / 8; + uint64_t PtrOff = ShAmt / bitsPerAU(); // For big endian targets, we need to adjust the offset to the pointer to // load the correct bytes. if (DAG.getDataLayout().isBigEndian()) - PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff; + PtrOff = inAUs(BitWidth - NewBW) - PtrOff; unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff); Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext()); @@ -14933,7 +14955,7 @@ Attribute::NoImplicitFloat); // This function cannot currently deal with non-byte-sized memory sizes. - if (ElementSizeBytes * 8 != MemVT.getSizeInBits()) + if (ElementSizeBytes * bitsPerAU() != MemVT.getSizeInBits()) return false; if (!MemVT.isSimple()) @@ -15038,7 +15060,7 @@ NonZero |= !IsElementZero; // Find a legal type for the constant store. - unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8; + unsigned SizeInBits = (i + 1) * ElementSizeBytes * bitsPerAU(); EVT StoreTy = EVT::getIntegerVT(Context, SizeInBits); bool IsFast = false; @@ -15293,7 +15315,7 @@ } // Find a legal type for the integer store. - unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8; + unsigned SizeInBits = (i + 1) * ElementSizeBytes * bitsPerAU(); StoreTy = EVT::getIntegerVT(Context, SizeInBits); if (TLI.isTypeLegal(StoreTy) && TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && @@ -15377,7 +15399,7 @@ unsigned Elts = NumElem * NumMemElts; JointMemOpVT = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts); } else { - unsigned SizeInBits = NumElem * ElementSizeBytes * 8; + unsigned SizeInBits = NumElem * ElementSizeBytes * bitsPerAU(); JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits); } @@ -15530,13 +15552,15 @@ MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); AAMDNodes AAInfo = ST->getAAInfo(); + // Calculate the offset in bytes. + const unsigned HalfSize = 32 / bitsPerAU(); SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(), ST->getAlignment(), MMOFlags, AAInfo); Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, - DAG.getConstant(4, DL, Ptr.getValueType())); - Alignment = MinAlign(Alignment, 4U); + DAG.getConstant(HalfSize, DL, Ptr.getValueType())); + Alignment = MinAlign(Alignment, HalfSize); SDValue St1 = DAG.getStore(Chain, DL, Hi, Ptr, - ST->getPointerInfo().getWithOffset(4), + ST->getPointerInfo().getWithOffset(HalfSize), Alignment, MMOFlags, AAInfo); return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, St0, St1); @@ -15792,7 +15816,8 @@ const BaseIndexOffset StoreBase = BaseIndexOffset::match(ST, DAG); // If we store purely within object bounds just before its lifetime ends, // we can remove the store. - if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase, + if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * bitsPerAU(), + StoreBase, ST->getMemoryVT().getStoreSizeInBits())) { LLVM_DEBUG(dbgs() << "\nRemoving store:"; StoreBase.dump(); dbgs() << "\nwithin LIFETIME_END of : "; @@ -15901,13 +15926,14 @@ // Lower value store. SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(), ST->getAlignment(), MMOFlags, AAInfo); + const unsigned HalfValByteSize = HalfValBitSize / bitsPerAU(); Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, - DAG.getConstant(HalfValBitSize / 8, DL, Ptr.getValueType())); + DAG.getConstant(HalfValByteSize, DL, Ptr.getValueType())); // Higher value store. SDValue St1 = DAG.getStore(St0, DL, Hi, Ptr, - ST->getPointerInfo().getWithOffset(HalfValBitSize / 8), + ST->getPointerInfo().getWithOffset(HalfValByteSize), Alignment / 2, MMOFlags, AAInfo); return St1; } @@ -16081,7 +16107,7 @@ SDLoc DL(EVE); if (auto *ConstEltNo = dyn_cast(EltNo)) { int Elt = ConstEltNo->getZExtValue(); - unsigned PtrOff = VecEltVT.getSizeInBits() * Elt / 8; + unsigned PtrOff = VecEltVT.getSizeInBits() * Elt / bitsPerAU(); Offset = DAG.getConstant(PtrOff, DL, PtrType); MPI = OriginalLoad->getPointerInfo().getWithOffset(PtrOff); } else { @@ -18768,8 +18794,8 @@ // Determine maximum split level (byte level masking). int MaxSplit = 1; - if (RVT.getScalarSizeInBits() % 8 == 0) - MaxSplit = RVT.getScalarSizeInBits() / 8; + if (RVT.getScalarSizeInBits() % bitsPerAU() == 0) + MaxSplit = RVT.getScalarSizeInBits() / bitsPerAU(); for (int Split = 1; Split <= MaxSplit; ++Split) if (RVT.getScalarSizeInBits() % Split == 0) @@ -19976,7 +20002,7 @@ return false; // Add ST's interval. - Intervals.insert(0, (St->getMemoryVT().getSizeInBits() + 7) / 8, Unit); + Intervals.insert(0, inAUs(St->getMemoryVT().getSizeInBits()), Unit); while (StoreSDNode *Chain = dyn_cast(STChain->getChain())) { // If the chain has more than one use, then we can't reorder the mem ops. @@ -19991,7 +20017,7 @@ int64_t Offset; if (!BasePtr.equalBaseIndex(Ptr, DAG, Offset)) break; - int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8; + int64_t Length = inAUs(Chain->getMemoryVT().getSizeInBits()); // Make sure we don't overlap with other intervals by checking the ones to // the left or right before inserting. auto I = Intervals.find(Offset); Index: lib/IR/DataLayout.cpp =================================================================== --- lib/IR/DataLayout.cpp +++ lib/IR/DataLayout.cpp @@ -185,6 +185,7 @@ StackNaturalAlign = 0; ProgramAddrSpace = 0; FunctionPtrAlign = 0; + BitsPerAddressableUnit = 8; TheFunctionPtrAlignType = FunctionPtrAlignType::Independent; ManglingMode = MM_None; NonIntegralAddressSpaces.clear();