Index: include/llvm/CodeGen/MachineMemOperand.h =================================================================== --- include/llvm/CodeGen/MachineMemOperand.h +++ include/llvm/CodeGen/MachineMemOperand.h @@ -16,6 +16,7 @@ #ifndef LLVM_CODEGEN_MACHINEMEMOPERAND_H #define LLVM_CODEGEN_MACHINEMEMOPERAND_H +#include "llvm/ADT/FlagsEnum.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/IR/Metadata.h" @@ -87,15 +88,20 @@ /// that aren't explicit in the regular LLVM IR. /// class MachineMemOperand { - MachinePointerInfo PtrInfo; - uint64_t Size; - unsigned Flags; - AAMDNodes AAInfo; - const MDNode *Ranges; - public: - /// Flags values. These may be or'd together. - enum MemOperandFlags { + // This is the number of bits we need to represent flags. + static constexpr unsigned MOMaxBits = 8; + + // Target hints allow target passes to annotate memory operations. + static constexpr unsigned MOTargetStartBit = 5; + static constexpr unsigned MOTargetNumBits = 3; + + /// Flags values. These may be or'd together. You can perform bitwise + /// operations on this enum without static_cast, because we've opted in below + /// via ENABLE_FLAGS_ENUM. + enum Flags : uint16_t { + // No flags set. + MONone = 0, /// The memory access reads data. MOLoad = 1, /// The memory access writes data. @@ -106,16 +112,26 @@ MONonTemporal = 8, /// The memory access is invariant. MOInvariant = 16, - // Target hints allow target passes to annotate memory operations. - MOTargetStartBit = 5, - MOTargetNumBits = 3, - // This is the number of bits we need to represent flags. - MOMaxBits = 8 + + // Maximum MemOperandFlag value (inclusive). This is needed for + // correctness, because the compiler can assume that we don't write to bits + // in this enum above the highest-order one that's set by one of the enum's + // values. Must be one less than a power of 2. + MOMaxFlag = (1 << MOMaxBits) - 1, }; +private: + MachinePointerInfo PtrInfo; + uint64_t Size; + Flags FlagVals; + uint16_t BaseAlignLog2; // log_2(base_alignment) + 1 + AAMDNodes AAInfo; + const MDNode *Ranges; + +public: /// Construct a MachineMemOperand object with the specified PtrInfo, flags, /// size, and base alignment. - MachineMemOperand(MachinePointerInfo PtrInfo, unsigned flags, uint64_t s, + MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr); @@ -137,11 +153,11 @@ const void *getOpaqueValue() const { return PtrInfo.V.getOpaqueValue(); } - /// Return the raw flags of the source value, \see MemOperandFlags. - unsigned int getFlags() const { return Flags & ((1 << MOMaxBits) - 1); } + /// Return the raw flags of the source value, \see Flags. + Flags getFlags() const { return FlagVals; } /// Bitwise OR the current flags with the given flags. - void setFlags(unsigned f) { Flags |= (f & ((1 << MOMaxBits) - 1)); } + void setFlags(Flags f); /// For normal values, this is a byte offset added to the base address. /// For PseudoSourceValue::FPRel values, this is the FrameIndex number. @@ -158,7 +174,7 @@ /// Return the minimum known alignment in bytes of the base address, without /// the offset. - uint64_t getBaseAlignment() const { return (1u << (Flags >> MOMaxBits)) >> 1; } + uint64_t getBaseAlignment() const { return (1u << BaseAlignLog2) >> 1; } /// Return the AA tags for the memory reference. AAMDNodes getAAInfo() const { return AAInfo; } @@ -166,11 +182,11 @@ /// Return the range tag for the memory reference. const MDNode *getRanges() const { return Ranges; } - bool isLoad() const { return Flags & MOLoad; } - bool isStore() const { return Flags & MOStore; } - bool isVolatile() const { return Flags & MOVolatile; } - bool isNonTemporal() const { return Flags & MONonTemporal; } - bool isInvariant() const { return Flags & MOInvariant; } + bool isLoad() const { return FlagVals & MOLoad; } + bool isStore() const { return FlagVals & MOStore; } + bool isVolatile() const { return FlagVals & MOVolatile; } + bool isNonTemporal() const { return FlagVals & MONonTemporal; } + bool isInvariant() const { return FlagVals & MOInvariant; } /// Returns true if this memory operation doesn't have any ordering /// constraints other than normal aliasing. Volatile and atomic memory @@ -222,6 +238,11 @@ } }; +ENABLE_FLAGS_ENUM(MachineMemOperand::Flags, MachineMemOperand::MOMaxFlag); + +// Needs to appear after the ENABLE_FLAGS_ENUM. +inline void MachineMemOperand::setFlags(Flags f) { FlagVals |= f; } + inline raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO) { MRO.print(OS); return OS; Index: lib/CodeGen/MachineFunction.cpp =================================================================== --- lib/CodeGen/MachineFunction.cpp +++ lib/CodeGen/MachineFunction.cpp @@ -299,8 +299,11 @@ uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges) { - return new (Allocator) MachineMemOperand(PtrInfo, f, s, base_alignment, - AAInfo, Ranges); + // FIXME: Get rid of this static_cast and make getMachineOperand take a + // MachineMemOperand::Flags param. + return new (Allocator) + MachineMemOperand(PtrInfo, static_cast(f), s, + base_alignment, AAInfo, Ranges); } MachineMemOperand * Index: lib/CodeGen/MachineInstr.cpp =================================================================== --- lib/CodeGen/MachineInstr.cpp +++ lib/CodeGen/MachineInstr.cpp @@ -497,13 +497,14 @@ return MachinePointerInfo(MF.getPSVManager().getStack(), Offset); } -MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, unsigned f, +MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, uint64_t s, unsigned int a, const AAMDNodes &AAInfo, const MDNode *Ranges) - : PtrInfo(ptrinfo), Size(s), - Flags((f & ((1 << MOMaxBits) - 1)) | ((Log2_32(a) + 1) << MOMaxBits)), - AAInfo(AAInfo), Ranges(Ranges) { + : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlignLog2(Log2_32(a) + 1), + AAInfo(AAInfo), Ranges(Ranges) { + assert(MOMaxFlag == (1 << MOMaxBits) - 1 && + "MOMaxFlag and MOMaxBits have fallen out of sync."); assert((PtrInfo.V.isNull() || PtrInfo.V.is() || isa(PtrInfo.V.get()->getType())) && "invalid pointer value"); @@ -517,7 +518,8 @@ ID.AddInteger(getOffset()); ID.AddInteger(Size); ID.AddPointer(getOpaqueValue()); - ID.AddInteger(Flags); + ID.AddInteger(getFlags()); + ID.AddInteger(getBaseAlignment()); } void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) { @@ -528,8 +530,7 @@ if (MMO->getBaseAlignment() >= getBaseAlignment()) { // Update the alignment value. - Flags = (Flags & ((1 << MOMaxBits) - 1)) | - ((Log2_32(MMO->getBaseAlignment()) + 1) << MOMaxBits); + BaseAlignLog2 = Log2_32(MMO->getBaseAlignment()) + 1; // Also update the base and offset, because the new alignment may // not be applicable with the old ones. PtrInfo = MMO->PtrInfo; Index: lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.cpp +++ lib/Target/AArch64/AArch64InstrInfo.cpp @@ -1463,7 +1463,8 @@ static_assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits), "Too many target MO flags"); (*MI.memoperands_begin()) - ->setFlags(MOSuppressPair << MachineMemOperand::MOTargetStartBit); + ->setFlags(static_cast( + MOSuppressPair << MachineMemOperand::MOTargetStartBit)); } bool AArch64InstrInfo::isUnscaledLdSt(unsigned Opc) const {