Index: llvm/include/llvm/CodeGen/MachineFunction.h =================================================================== --- llvm/include/llvm/CodeGen/MachineFunction.h +++ llvm/include/llvm/CodeGen/MachineFunction.h @@ -847,6 +847,13 @@ AtomicOrdering Ordering = AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); + MachineMemOperand *getMachineMemOperand( + MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, + Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), + const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System, + AtomicOrdering Ordering = AtomicOrdering::NotAtomic, + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); + /// getMachineMemOperand - Allocate a new MachineMemOperand by copying /// an existing one, adjusting by an offset and using the given size. /// MachineMemOperands are owned by the MachineFunction and need not be Index: llvm/include/llvm/CodeGen/MachineMemOperand.h =================================================================== --- llvm/include/llvm/CodeGen/MachineMemOperand.h +++ llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -22,6 +22,7 @@ #include "llvm/IR/Value.h" // PointerLikeTypeTraits #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/LowLevelTypeImpl.h" namespace llvm { @@ -168,7 +169,11 @@ }; MachinePointerInfo PtrInfo; - uint64_t Size; + + /// Track the memory type of the access. An access size which is unknown or + /// too large to be represented by LLT should use the invalid LLT. + LLT MemoryType; + Flags FlagVals; Align BaseAlign; MachineAtomicInfo AtomicInfo; @@ -187,6 +192,12 @@ SyncScope::ID SSID = SyncScope::System, AtomicOrdering Ordering = AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); + MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, + Align a, const AAMDNodes &AAInfo = AAMDNodes(), + const MDNode *Ranges = nullptr, + SyncScope::ID SSID = SyncScope::System, + AtomicOrdering Ordering = AtomicOrdering::NotAtomic, + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); const MachinePointerInfo &getPointerInfo() const { return PtrInfo; } @@ -217,11 +228,23 @@ unsigned getAddrSpace() const { return PtrInfo.getAddrSpace(); } + /// Return the memory type of the memory reference. This should only be relied + /// on for GlobalISel G_* operation legalization. + LLT getMemoryType() const { return MemoryType; } + /// Return the size in bytes of the memory reference. - uint64_t getSize() const { return Size; } + uint64_t getSize() const { + return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0); + } /// Return the size in bits of the memory reference. - uint64_t getSizeInBits() const { return Size * 8; } + uint64_t getSizeInBits() const { + return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0); + } + + LLT getType() const { + return MemoryType; + } /// Return the minimum known alignment in bytes of the actual memory /// reference. Index: llvm/include/llvm/Support/LowLevelTypeImpl.h =================================================================== --- llvm/include/llvm/Support/LowLevelTypeImpl.h +++ llvm/include/llvm/Support/LowLevelTypeImpl.h @@ -296,6 +296,7 @@ } } +public: uint64_t getUniqueRAWLLTData() const { return ((uint64_t)RawData) << 2 | ((uint64_t)IsPointer) << 1 | ((uint64_t)IsVector); Index: llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1296,7 +1296,7 @@ AAMDNodes AAMetadata; LI.getAAMetadata(AAMetadata); auto MMO = MF->getMachineMemOperand( - Ptr, Flags, MRI->getType(Regs[i]).getSizeInBytes(), + Ptr, Flags, MRI->getType(Regs[i]), commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges, LI.getSyncScopeID(), LI.getOrdering()); MIRBuilder.buildLoad(Regs[i], Addr, *MMO); @@ -1338,7 +1338,7 @@ AAMDNodes AAMetadata; SI.getAAMetadata(AAMetadata); auto MMO = MF->getMachineMemOperand( - Ptr, Flags, MRI->getType(Vals[i]).getSizeInBytes(), + Ptr, Flags, MRI->getType(Vals[i]), commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr, SI.getSyncScopeID(), SI.getOrdering()); MIRBuilder.buildStore(Vals[i], Addr, *MMO); @@ -2052,7 +2052,7 @@ *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore | MachineMemOperand::MOVolatile, - PtrTy.getSizeInBits() / 8, Align(8))); + PtrTy, Align(8))); return true; } case Intrinsic::stacksave: { @@ -2338,10 +2338,10 @@ if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) { Align Alignment = Info.align.getValueOr( DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext()))); - - uint64_t Size = Info.memVT.getStoreSize(); + LLT MemTy = Info.memVT.isSimple() ? getLLTForMVT(Info.memVT.getSimpleVT()) : + LLT::scalar(Info.memVT.getStoreSizeInBits()); MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal), - Info.flags, Size, Alignment)); + Info.flags, MemTy, Alignment)); } return true; @@ -2707,9 +2707,6 @@ auto &TLI = *MF->getSubtarget().getTargetLowering(); auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); - Type *ResType = I.getType(); - Type *ValType = ResType->Type::getStructElementType(0); - auto Res = getOrCreateVRegs(I); Register OldValRes = Res[0]; Register SuccessRes = Res[1]; @@ -2724,7 +2721,7 @@ OldValRes, SuccessRes, Addr, Cmp, NewVal, *MF->getMachineMemOperand( MachinePointerInfo(I.getPointerOperand()), Flags, - DL->getTypeStoreSize(ValType), getMemOpAlign(I), AAMetadata, nullptr, + MRI->getType(Cmp), getMemOpAlign(I), AAMetadata, nullptr, I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering())); return true; } @@ -2735,8 +2732,6 @@ auto &TLI = *MF->getSubtarget().getTargetLowering(); auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); - Type *ResType = I.getType(); - Register Res = getOrCreateVReg(I); Register Addr = getOrCreateVReg(*I.getPointerOperand()); Register Val = getOrCreateVReg(*I.getValOperand()); @@ -2792,7 +2787,7 @@ MIRBuilder.buildAtomicRMW( Opcode, Res, Addr, Val, *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), - Flags, DL->getTypeStoreSize(ResType), + Flags, MRI->getType(Val), getMemOpAlign(I), AAMetadata, nullptr, I.getSyncScopeID(), I.getOrdering())); return true; Index: llvm/lib/CodeGen/LowLevelType.cpp =================================================================== --- llvm/lib/CodeGen/LowLevelType.cpp +++ llvm/lib/CodeGen/LowLevelType.cpp @@ -56,8 +56,8 @@ if (!Ty.isVector()) return LLT::scalar(Ty.getSizeInBits()); - return LLT::vector(Ty.getVectorNumElements(), - Ty.getVectorElementType().getSizeInBits()); + return LLT::scalarOrVector(Ty.getVectorNumElements(), + Ty.getVectorElementType().getSizeInBits()); } const llvm::fltSemantics &llvm::getFltSemanticForLLT(LLT Ty) { Index: llvm/lib/CodeGen/MachineFunction.cpp =================================================================== --- llvm/lib/CodeGen/MachineFunction.cpp +++ llvm/lib/CodeGen/MachineFunction.cpp @@ -437,6 +437,16 @@ SSID, Ordering, FailureOrdering); } +MachineMemOperand *MachineFunction::getMachineMemOperand( + MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, + Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges, + SyncScope::ID SSID, AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) { + return new (Allocator) + MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges, + SSID, Ordering, FailureOrdering); +} + MachineMemOperand *MachineFunction::getMachineMemOperand( const MachineMemOperand *MMO, MachinePointerInfo &PtrInfo, uint64_t Size) { return new (Allocator) MachineMemOperand( Index: llvm/lib/CodeGen/MachineOperand.cpp =================================================================== --- llvm/lib/CodeGen/MachineOperand.cpp +++ llvm/lib/CodeGen/MachineOperand.cpp @@ -1015,12 +1015,12 @@ } MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, - uint64_t s, Align a, + LLT type, Align a, const AAMDNodes &AAInfo, const MDNode *Ranges, SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) - : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlign(a), AAInfo(AAInfo), + : PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a), AAInfo(AAInfo), Ranges(Ranges) { assert((PtrInfo.V.isNull() || PtrInfo.V.is() || isa(PtrInfo.V.get()->getType())) && @@ -1035,11 +1035,22 @@ assert(getFailureOrdering() == FailureOrdering && "Value truncated"); } +MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, + uint64_t s, Align a, + const AAMDNodes &AAInfo, + const MDNode *Ranges, SyncScope::ID SSID, + AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) + : MachineMemOperand(ptrinfo, f, + s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a, + AAInfo, Ranges, SSID, Ordering, + FailureOrdering) {} + /// Profile - Gather unique data for the object. /// void MachineMemOperand::Profile(FoldingSetNodeID &ID) const { ID.AddInteger(getOffset()); - ID.AddInteger(Size); + ID.AddInteger(getMemoryType().getUniqueRAWLLTData()); ID.AddPointer(getOpaqueValue()); ID.AddInteger(getFlags()); ID.AddInteger(getBaseAlign().value());