diff --git a/llvm/include/llvm/AsmParser/SlotMapping.h b/llvm/include/llvm/AsmParser/SlotMapping.h --- a/llvm/include/llvm/AsmParser/SlotMapping.h +++ b/llvm/include/llvm/AsmParser/SlotMapping.h @@ -32,6 +32,7 @@ struct SlotMapping { std::vector GlobalValues; std::map MetadataNodes; + std::map ForwardRefMDNodes; StringMap NamedTypes; std::map Types; }; diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h --- a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h +++ b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h @@ -161,7 +161,7 @@ BumpPtrAllocator Allocator; MachineFunction &MF; SourceMgr *SM; - const SlotMapping &IRSlots; + SlotMapping &IRSlots; PerTargetMIParsingState &Target; DenseMap MBBSlots; @@ -176,7 +176,7 @@ DenseMap Slots2Values; PerFunctionMIParsingState(MachineFunction &MF, SourceMgr &SM, - const SlotMapping &IRSlots, + SlotMapping &IRSlots, PerTargetMIParsingState &Target); VRegInfo &getVRegInfo(Register Num); @@ -233,6 +233,9 @@ bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node, StringRef Src, SMDiagnostic &Error); +bool parseStandaloneMetadata(PerFunctionMIParsingState &PFS, StringRef Src, + SMDiagnostic &Error); + } // end namespace llvm #endif // LLVM_CODEGEN_MIRPARSER_MIPARSER_H diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h --- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -704,6 +704,7 @@ std::vector CallSitesInfo; std::vector DebugValueSubstitutions; MachineJumpTable JumpTableInfo; + std::vector MachineMetadataNodes; BlockStringValue Body; }; @@ -738,6 +739,9 @@ YamlIO.mapOptional("machineFunctionInfo", MF.MachineFuncInfo); if (!YamlIO.outputting() || !MF.JumpTableInfo.Entries.empty()) YamlIO.mapOptional("jumpTable", MF.JumpTableInfo, MachineJumpTable()); + if (!YamlIO.outputting() || !MF.MachineMetadataNodes.empty()) + YamlIO.mapOptional("machineMetadataNodes", MF.MachineMetadataNodes, + std::vector()); YamlIO.mapOptional("body", MF.Body, BlockStringValue()); } }; diff --git a/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h b/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineModuleSlotTracker.h @@ -0,0 +1,38 @@ +#ifndef LLVM_CODEGEN_MACHINEMODULESLOTTRACKER_H +#define LLVM_CODEGEN_MACHINEMODULESLOTTRACKER_H + +#include "llvm/IR/ModuleSlotTracker.h" + +namespace llvm { + +class AbstractSlotTrackerStorage; +class Function; +class MachineModuleInfo; +class MachineFunction; +class Module; + +class MachineModuleSlotTracker : public ModuleSlotTracker { + const MachineModuleInfo &MMI; + unsigned MDNStartSlot, MDNEndSlot; + + void processMachineFunctionMetadata(AbstractSlotTrackerStorage *AST, + const MachineFunction &MF); + void processMachineModule(AbstractSlotTrackerStorage *AST, const Module *M, + bool ShouldInitializeAllMetadata); + void processMachineFunction(AbstractSlotTrackerStorage *AST, + const Function *F, + bool ShouldInitializeAllMetadata); + +public: + MachineModuleSlotTracker(const MachineModuleInfo &MMI, const Module *M, + bool ShouldInitializeAllMetadata = true); + MachineModuleSlotTracker(const MachineFunction *MF, + bool ShouldInitializeAllMetadata = true); + ~MachineModuleSlotTracker(); + + void collectMachineMDNodes(MachineMDNodeListType &L) const; +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_MACHINEMODULESLOTTRACKER_H diff --git a/llvm/include/llvm/IR/ModuleSlotTracker.h b/llvm/include/llvm/IR/ModuleSlotTracker.h --- a/llvm/include/llvm/IR/ModuleSlotTracker.h +++ b/llvm/include/llvm/IR/ModuleSlotTracker.h @@ -9,7 +9,10 @@ #ifndef LLVM_IR_MODULESLOTTRACKER_H #define LLVM_IR_MODULESLOTTRACKER_H +#include #include +#include +#include namespace llvm { @@ -17,6 +20,18 @@ class Function; class SlotTracker; class Value; +class MDNode; + +/// Abstract interface of slot tracker storage. +class AbstractSlotTrackerStorage { +public: + virtual ~AbstractSlotTrackerStorage(); + + virtual unsigned getNextMetadataSlot() = 0; + + virtual void createMetadataSlot(const MDNode *) = 0; + virtual int getMetadataSlot(const MDNode *) = 0; +}; /// Manage lifetime of a slot tracker for printing IR. /// @@ -36,6 +51,11 @@ const Function *F = nullptr; SlotTracker *Machine = nullptr; + std::function + processModuleHookFn; + std::function + processFunctionHookFn; + public: /// Wrap a preinitialized SlotTracker. ModuleSlotTracker(SlotTracker &Machine, const Module *M, @@ -52,7 +72,7 @@ bool ShouldInitializeAllMetadata = true); /// Destructor to clean up storage. - ~ModuleSlotTracker(); + virtual ~ModuleSlotTracker(); /// Lazily creates a slot tracker. SlotTracker *getMachine(); @@ -72,6 +92,16 @@ /// this method. /// Return -1 if the value is not in the function's SlotTracker. int getLocalSlot(const Value *V); + + void setProcessHook( + std::function); + void setProcessHook(std::function); + + using MachineMDNodeListType = + std::vector>; + + void collectMDNodes(MachineMDNodeListType &L, unsigned LB, unsigned UB) const; }; } // end namespace llvm diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -91,6 +91,7 @@ MachineLoopUtils.cpp MachineModuleInfo.cpp MachineModuleInfoImpls.cpp + MachineModuleSlotTracker.cpp MachineOperand.cpp MachineOptimizationRemarkEmitter.cpp MachineOutliner.cpp diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h --- a/llvm/lib/CodeGen/MIRParser/MILexer.h +++ b/llvm/lib/CodeGen/MIRParser/MILexer.h @@ -128,6 +128,10 @@ kw_unknown_size, kw_unknown_address, + // Metadata types. + kw_distinct, + kw_null, + // Named metadata keywords md_tbaa, md_alias_scope, diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp --- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -272,6 +272,8 @@ .Case("bbsections", MIToken::kw_bbsections) .Case("unknown-size", MIToken::kw_unknown_size) .Case("unknown-address", MIToken::kw_unknown_address) + .Case("distinct", MIToken::kw_distinct) + .Case("null", MIToken::kw_null) .Default(MIToken::Identifier); } diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -317,9 +317,10 @@ } PerFunctionMIParsingState::PerFunctionMIParsingState(MachineFunction &MF, - SourceMgr &SM, const SlotMapping &IRSlots, PerTargetMIParsingState &T) - : MF(MF), SM(&SM), IRSlots(IRSlots), Target(T) { -} + SourceMgr &SM, + SlotMapping &IRSlots, + PerTargetMIParsingState &T) + : MF(MF), SM(&SM), IRSlots(IRSlots), Target(T) {} VRegInfo &PerFunctionMIParsingState::getVRegInfo(Register Num) { auto I = VRegInfos.insert(std::make_pair(Num, nullptr)); @@ -428,6 +429,10 @@ bool parseStandaloneRegister(Register &Reg); bool parseStandaloneStackObject(int &FI); bool parseStandaloneMDNode(MDNode *&Node); + bool parseStandaloneMetadata(); + bool parseMDTuple(MDNode *&MD, bool IsDistinct); + bool parseMDNodeVector(SmallVectorImpl &Elts); + bool parseMetadata(Metadata *&MD); bool parseBasicBlockDefinition(DenseMap &MBBSlots); @@ -1170,6 +1175,127 @@ return false; } +bool MIParser::parseStandaloneMetadata() { + lex(); + if (Token.isNot(MIToken::exclaim)) + return error("expected a metadata node"); + + lex(); + if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned()) + return error("expected metadata id after '!'"); + unsigned ID = 0; + if (getUnsigned(ID)) + return true; + lex(); + if (expectAndConsume(MIToken::equal)) + return true; + bool IsDistinct = Token.is(MIToken::kw_distinct); + if (IsDistinct) + lex(); + if (Token.isNot(MIToken::exclaim)) + return error("expected a metadata node"); + lex(); + + MDNode *MD; + if (parseMDTuple(MD, IsDistinct)) + return true; + + auto FI = PFS.IRSlots.ForwardRefMDNodes.find(ID); + if (FI != PFS.IRSlots.ForwardRefMDNodes.end()) { + FI->second->replaceAllUsesWith(MD); + PFS.IRSlots.ForwardRefMDNodes.erase(FI); + + assert(PFS.IRSlots.MetadataNodes[ID] == MD && "Tracking VH didn't work"); + } else { + if (PFS.IRSlots.MetadataNodes.count(ID)) + return error("Metadata id is already used"); + PFS.IRSlots.MetadataNodes[ID].reset(MD); + } + + return false; +} + +bool MIParser::parseMDTuple(MDNode *&MD, bool IsDistinct) { + SmallVector Elts; + if (parseMDNodeVector(Elts)) + return true; + MD = (IsDistinct ? MDTuple::getDistinct + : MDTuple::get)(MF.getFunction().getContext(), Elts); + return false; +} + +bool MIParser::parseMDNodeVector(SmallVectorImpl &Elts) { + if (Token.isNot(MIToken::lbrace)) + return error("expected '{' here"); + lex(); + + if (Token.is(MIToken::rbrace)) { + lex(); + return false; + } + + do { + if (Token.is(MIToken::kw_null)) { + lex(); + Elts.push_back(nullptr); + continue; + } + + Metadata *MD; + if (parseMetadata(MD)) + return true; + + Elts.push_back(MD); + + if (Token.isNot(MIToken::comma)) + break; + lex(); + } while (true); + + if (Token.isNot(MIToken::rbrace)) + return error("expected end of metadata node"); + lex(); + + return false; +} + +// ::= !42 +// ::= !"string" +bool MIParser::parseMetadata(Metadata *&MD) { + if (Token.isNot(MIToken::exclaim)) + return error("expected '!' here"); + lex(); + + if (Token.is(MIToken::StringConstant)) { + std::string Str; + if (parseStringConstant(Str)) + return true; + MD = MDString::get(MF.getFunction().getContext(), Str); + return false; + } + + if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned()) + return error("expected metadata id after '!'"); + + unsigned ID = 0; + if (getUnsigned(ID)) + return true; + lex(); + + auto NodeInfo = PFS.IRSlots.MetadataNodes.find(ID); + if (NodeInfo != PFS.IRSlots.MetadataNodes.end()) { + MD = NodeInfo->second.get(); + return false; + } + // Forward reference. + auto &FwdRef = PFS.IRSlots.ForwardRefMDNodes[ID]; + FwdRef = MDTuple::getTemporary(MF.getFunction().getContext(), None); + PFS.IRSlots.MetadataNodes[ID].reset(FwdRef.get()); + MD = FwdRef.get(); + + return false; +} + static const char *printImplicitRegisterFlag(const MachineOperand &MO) { assert(MO.isImplicit()); return MO.isDef() ? "implicit-def" : "implicit"; @@ -3258,6 +3384,11 @@ return MIParser(PFS, Error, Src).parseStandaloneMDNode(Node); } +bool llvm::parseStandaloneMetadata(PerFunctionMIParsingState &PFS, + StringRef Src, SMDiagnostic &Error) { + return MIParser(PFS, Error, Src).parseStandaloneMetadata(); +} + bool MIRFormatter::parseIRValue(StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS, const Value *&V, ErrorCallbackType ErrorCallback) { diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp --- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -143,6 +143,10 @@ bool initializeJumpTableInfo(PerFunctionMIParsingState &PFS, const yaml::MachineJumpTable &YamlJTI); + bool parseMachineMetadataNodes(PerFunctionMIParsingState &PFS, + MachineFunction &MF, + const yaml::MachineFunction &YMF); + private: bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node, const yaml::StringValue &Source); @@ -151,6 +155,9 @@ MachineBasicBlock *&MBB, const yaml::StringValue &Source); + bool parseStandaloneMetadata(PerFunctionMIParsingState &PFS, + const yaml::StringValue &Source); + /// Return a MIR diagnostic converted from an MI string diagnostic. SMDiagnostic diagFromMIStringDiag(const SMDiagnostic &Error, SMRange SourceRange); @@ -457,6 +464,9 @@ if (initializeConstantPool(PFS, *ConstantPool, YamlMF)) return true; } + if (!YamlMF.MachineMetadataNodes.empty() && + parseMachineMetadataNodes(PFS, MF, YamlMF)) + return true; StringRef BlockStr = YamlMF.Body.Value.Value; SMDiagnostic Error; @@ -920,6 +930,24 @@ return false; } +bool MIRParserImpl::parseStandaloneMetadata(PerFunctionMIParsingState &PFS, + const yaml::StringValue &Source) { + SMDiagnostic Error; + if (llvm::parseStandaloneMetadata(PFS, Source.Value, Error)) + return error(Error, Source.SourceRange); + return false; +} + +bool MIRParserImpl::parseMachineMetadataNodes( + PerFunctionMIParsingState &PFS, MachineFunction &MF, + const yaml::MachineFunction &YMF) { + for (auto &MDS : YMF.MachineMetadataNodes) { + if (parseStandaloneMetadata(PFS, MDS)) + return true; + } + return false; +} + SMDiagnostic MIRParserImpl::diagFromMIStringDiag(const SMDiagnostic &Error, SMRange SourceRange) { assert(SourceRange.isValid() && "Invalid source range"); diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -29,6 +29,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineModuleSlotTracker.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" @@ -135,6 +136,9 @@ void convertCallSiteObjects(yaml::MachineFunction &YMF, const MachineFunction &MF, ModuleSlotTracker &MST); + void convertMachineMetadataNodes(yaml::MachineFunction &YMF, + const MachineFunction &MF, + MachineModuleSlotTracker &MST); private: void initRegisterMaskIds(const MachineFunction &MF); @@ -215,7 +219,7 @@ MachineFunctionProperties::Property::FailedISel); convert(YamlMF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo()); - ModuleSlotTracker MST(MF.getFunction().getParent()); + MachineModuleSlotTracker MST(&MF); MST.incorporateFunction(MF.getFunction()); convert(MST, YamlMF.FrameInfo, MF.getFrameInfo()); convertStackObjects(YamlMF, MF, MST); @@ -243,6 +247,10 @@ IsNewlineNeeded = true; } StrOS.flush(); + // Convert machine metadata collected during the print of the machine + // function. + convertMachineMetadataNodes(YamlMF, MF, MST); + yaml::Output Out(OS); if (!SimplifyMIR) Out.setWriteDefaultValues(true); @@ -525,6 +533,19 @@ }); } +void MIRPrinter::convertMachineMetadataNodes(yaml::MachineFunction &YMF, + const MachineFunction &MF, + MachineModuleSlotTracker &MST) { + MachineModuleSlotTracker::MachineMDNodeListType MDList; + MST.collectMachineMDNodes(MDList); + for (auto &MD : MDList) { + std::string NS; + raw_string_ostream StrOS(NS); + MD.second->print(StrOS, MST, MF.getFunction().getParent()); + YMF.MachineMetadataNodes.push_back(StrOS.str()); + } +} + void MIRPrinter::convert(yaml::MachineFunction &MF, const MachineConstantPool &ConstantPool) { unsigned ID = 0; diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -30,6 +30,7 @@ #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineModuleSlotTracker.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/CodeGen/TargetFrameLowering.h" @@ -52,7 +53,6 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" -#include "llvm/IR/ModuleSlotTracker.h" #include "llvm/IR/Value.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCSymbol.h" @@ -553,7 +553,7 @@ OS << '\n'; } - ModuleSlotTracker MST(getFunction().getParent()); + MachineModuleSlotTracker MST(this); MST.incorporateFunction(getFunction()); for (const auto &BB : *this) { OS << '\n'; diff --git a/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp b/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp @@ -0,0 +1,74 @@ +#include "llvm/CodeGen/MachineModuleSlotTracker.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineModuleInfo.h" + +using namespace llvm; + +void MachineModuleSlotTracker::processMachineFunctionMetadata( + AbstractSlotTrackerStorage *AST, const MachineFunction &MF) { + // Create metadata created within the backend. + for (const MachineBasicBlock &MBB : MF) + for (const MachineInstr &MI : MBB.instrs()) + for (const MachineMemOperand *MMO : MI.memoperands()) { + AAMDNodes AAInfo = MMO->getAAInfo(); + if (AAInfo.TBAA) + AST->createMetadataSlot(AAInfo.TBAA); + if (AAInfo.TBAAStruct) + AST->createMetadataSlot(AAInfo.TBAAStruct); + if (AAInfo.Scope) + AST->createMetadataSlot(AAInfo.Scope); + if (AAInfo.NoAlias) + AST->createMetadataSlot(AAInfo.NoAlias); + } +} + +void MachineModuleSlotTracker::processMachineModule( + AbstractSlotTrackerStorage *AST, const Module *M, + bool ShouldInitializeAllMetadata) { + if (ShouldInitializeAllMetadata) { + MDNStartSlot = AST->getNextMetadataSlot(); + for (const Function &F : *M) { + if (auto MF = MMI.getMachineFunction(F)) + processMachineFunctionMetadata(AST, *MF); + } + MDNEndSlot = AST->getNextMetadataSlot(); + } +} + +void MachineModuleSlotTracker::processMachineFunction( + AbstractSlotTrackerStorage *AST, const Function *F, + bool ShouldInitializeAllMetadata) { + if (!ShouldInitializeAllMetadata) { + MDNStartSlot = AST->getNextMetadataSlot(); + if (auto MF = MMI.getMachineFunction(*F)) + processMachineFunctionMetadata(AST, *MF); + MDNEndSlot = AST->getNextMetadataSlot(); + } +} + +void MachineModuleSlotTracker::collectMachineMDNodes( + MachineMDNodeListType &L) const { + collectMDNodes(L, MDNStartSlot, MDNEndSlot); +} + +MachineModuleSlotTracker::MachineModuleSlotTracker( + const MachineModuleInfo &MMI, const Module *M, + bool ShouldInitializeAllMetadata) + : ModuleSlotTracker(M, ShouldInitializeAllMetadata), MMI(MMI), + MDNStartSlot(0), MDNEndSlot(0) { + setProcessHook([this](AbstractSlotTrackerStorage *AST, const Module *M, + bool ShouldInitializeAllMetadata) { + this->processMachineModule(AST, M, ShouldInitializeAllMetadata); + }); + setProcessHook([this](AbstractSlotTrackerStorage *AST, const Function *F, + bool ShouldInitializeAllMetadata) { + this->processMachineFunction(AST, F, ShouldInitializeAllMetadata); + }); +} + +MachineModuleSlotTracker::MachineModuleSlotTracker( + const MachineFunction *MF, bool ShouldInitializeAllMetadata) + : MachineModuleSlotTracker(MF->getMMI(), MF->getFunction().getParent(), + ShouldInitializeAllMetadata) {} + +MachineModuleSlotTracker::~MachineModuleSlotTracker() = default; diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp --- a/llvm/lib/CodeGen/MachineStableHash.cpp +++ b/llvm/lib/CodeGen/MachineStableHash.cpp @@ -31,7 +31,6 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/ModuleSlotTracker.h" #include "llvm/MC/MCDwarf.h" #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetMachine.h" diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -52,6 +52,7 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -97,6 +98,10 @@ #define DEBUG_TYPE "selectiondag" +static cl::opt EnableMemCpyScopedNoAlias( + "enable-memcpy-scoped-noalias", cl::Hidden, cl::init(true), + cl::desc("Enable scoped no-alias support during memcpy lowering")); + static cl::opt EnableMemCpyDAGOpt("enable-memcpy-dag-opt", cl::Hidden, cl::init(true), cl::desc("Gang up loads and stores generated by inlining of memcpy")); @@ -6391,6 +6396,30 @@ AAMDNodes NewAAInfo = AAInfo; NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; + AAMDNodes DstAAInfo, SrcAAInfo; + DstAAInfo = SrcAAInfo = NewAAInfo; + // Generate new scoped AA metadata for this memcpy instance if enabled. + if (EnableMemCpyScopedNoAlias) { + MDBuilder MDB(*DAG.getContext()); + MDNode *Domain = + MDB.createAnonymousAliasScopeDomain("MemcpyLoweringDomain"); + MDNode *DstScope = MDB.createAnonymousAliasScope(Domain, "Dst"); + MDNode *SrcScope = MDB.createAnonymousAliasScope(Domain, "Src"); + MDNode *DstAliasScope = MDNode::concatenate( + NewAAInfo.Scope, MDNode::get(*DAG.getContext(), {DstScope})); + MDNode *DstNoAliase = MDNode::concatenate( + NewAAInfo.NoAlias, MDNode::get(*DAG.getContext(), {SrcScope})); + MDNode *SrcAliasScope = MDNode::concatenate( + NewAAInfo.Scope, MDNode::get(*DAG.getContext(), {SrcScope})); + MDNode *SrcNoAliase = MDNode::concatenate( + NewAAInfo.NoAlias, MDNode::get(*DAG.getContext(), {DstScope})); + + DstAAInfo.Scope = DstAliasScope; + DstAAInfo.NoAlias = DstNoAliase; + SrcAAInfo.Scope = SrcAliasScope; + SrcAAInfo.NoAlias = SrcNoAliase; + } + MachineMemOperand::Flags MMOFlags = isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; SmallVector OutLoadChains; @@ -6433,7 +6462,7 @@ Store = DAG.getStore( Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, DstAAInfo); OutChains.push_back(Store); } } @@ -6457,13 +6486,13 @@ ISD::EXTLOAD, dl, NVT, Chain, DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), SrcPtrInfo.getWithOffset(SrcOff), VT, - commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo); + commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, SrcAAInfo); OutLoadChains.push_back(Value.getValue(1)); Store = DAG.getTruncStore( Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo); + DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, DstAAInfo); OutStoreChains.push_back(Store); } SrcOff += VTSize; diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -713,6 +713,8 @@ OS << '>'; } +AbstractSlotTrackerStorage::~AbstractSlotTrackerStorage() {} + namespace llvm { //===----------------------------------------------------------------------===// @@ -720,7 +722,7 @@ //===----------------------------------------------------------------------===// /// This class provides computation of slot numbers for LLVM Assembly writing. /// -class SlotTracker { +class SlotTracker : public AbstractSlotTrackerStorage { public: /// ValueMap - A mapping of Values to slot numbers. using ValueMap = DenseMap; @@ -734,6 +736,11 @@ bool FunctionProcessed = false; bool ShouldInitializeAllMetadata; + std::function + processModuleHookFn; + std::function + processFunctionHookFn; + /// The summary index for which we are holding slot numbers. const ModuleSummaryIndex *TheIndex = nullptr; @@ -788,11 +795,22 @@ SlotTracker(const SlotTracker &) = delete; SlotTracker &operator=(const SlotTracker &) = delete; + ~SlotTracker() = default; + + void setProcessHook( + std::function); + void setProcessHook(std::function); + + unsigned getNextMetadataSlot() override { return mdnNext; } + + void createMetadataSlot(const MDNode *N) override; + /// Return the slot number of the specified value in it's type /// plane. If something is not in the SlotTracker, return -1. int getLocalSlot(const Value *V); int getGlobalSlot(const GlobalValue *V); - int getMetadataSlot(const MDNode *N); + int getMetadataSlot(const MDNode *N) override; int getAttributeGroupSlot(AttributeSet AS); int getModulePathSlot(StringRef Path); int getGUIDSlot(GlobalValue::GUID GUID); @@ -893,6 +911,10 @@ MachineStorage = std::make_unique(M, ShouldInitializeAllMetadata); Machine = MachineStorage.get(); + if (processModuleHookFn) + Machine->setProcessHook(processModuleHookFn); + if (processFunctionHookFn) + Machine->setProcessHook(processFunctionHookFn); return Machine; } @@ -915,6 +937,18 @@ return Machine->getLocalSlot(V); } +void ModuleSlotTracker::setProcessHook( + std::function + Fn) { + processModuleHookFn = Fn; +} + +void ModuleSlotTracker::setProcessHook( + std::function + Fn) { + processFunctionHookFn = Fn; +} + static SlotTracker *createSlotTracker(const Value *V) { if (const Argument *FA = dyn_cast(V)) return new SlotTracker(FA->getParent()); @@ -1025,6 +1059,9 @@ CreateAttributeSetSlot(FnAttrs); } + if (processModuleHookFn) + processModuleHookFn(this, TheModule, ShouldInitializeAllMetadata); + ST_DEBUG("end processModule!\n"); } @@ -1065,6 +1102,9 @@ } } + if (processFunctionHookFn) + processFunctionHookFn(this, TheFunction, ShouldInitializeAllMetadata); + FunctionProcessed = true; ST_DEBUG("end processFunction!\n"); @@ -1155,6 +1195,23 @@ return MI == mMap.end() ? -1 : (int)MI->second; } +void SlotTracker::setProcessHook( + std::function + Fn) { + processModuleHookFn = Fn; +} + +void SlotTracker::setProcessHook( + std::function + Fn) { + processFunctionHookFn = Fn; +} + +/// getMetadataSlot - Get the slot number of a MDNode. +void SlotTracker::createMetadataSlot(const MDNode *N) { + CreateMetadataSlot(N); +} + /// getMetadataSlot - Get the slot number of a MDNode. int SlotTracker::getMetadataSlot(const MDNode *N) { // Check for uninitialized state and do lazy initialization. @@ -4767,6 +4824,17 @@ W.printModuleSummaryIndex(); } +void ModuleSlotTracker::collectMDNodes(MachineMDNodeListType &L, unsigned LB, + unsigned UB) const { + SlotTracker *ST = MachineStorage.get(); + if (!ST) + return; + + for (auto &I : llvm::make_range(ST->mdn_begin(), ST->mdn_end())) + if (I.second >= LB && I.second < UB) + L.push_back(std::make_pair(I.second, I.first)); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) // Value::dump - allow easy printing of Values from the debugger. LLVM_DUMP_METHOD diff --git a/llvm/test/CodeGen/AArch64/arm64-2012-05-07-MemcpyAlignBug.ll b/llvm/test/CodeGen/AArch64/arm64-2012-05-07-MemcpyAlignBug.ll --- a/llvm/test/CodeGen/AArch64/arm64-2012-05-07-MemcpyAlignBug.ll +++ b/llvm/test/CodeGen/AArch64/arm64-2012-05-07-MemcpyAlignBug.ll @@ -9,9 +9,9 @@ ; CHECK: adrp x[[PAGE:[0-9]+]], {{l_b@PAGE|.Lb}} ; CHECK: add x[[ADDR:[0-9]+]], x[[PAGE]], {{l_b@PAGEOFF|:lo12:.Lb}} ; CHECK-NEXT: ldr [[VAL2:x[0-9]+]], [x[[ADDR]]] +; CHECK-NEXT: str [[VAL2]], [x0] ; CHECK-NEXT: ldr [[VAL:w[0-9]+]], [x[[ADDR]], #8] ; CHECK-NEXT: str [[VAL]], [x0, #8] -; CHECK-NEXT: str [[VAL2]], [x0] define void @foo(i8* %a) { call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast ([3 x i32]* @b to i8*), i64 12, i1 false) diff --git a/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll b/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll --- a/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll +++ b/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll @@ -27,10 +27,10 @@ define void @t1(i8* nocapture %C) nounwind { entry: ; CHECK-LABEL: t1: -; CHECK: ldr [[DEST:q[0-9]+]], [x[[BASEREG]]] -; CHECK: ldur [[DEST:q[0-9]+]], [x[[BASEREG:[0-9]+]], #15] -; CHECK: stur [[DEST:q[0-9]+]], [x0, #15] -; CHECK: str [[DEST:q[0-9]+]], [x0] +; CHECK: ldr [[REG0:q[0-9]+]], [x[[BASEREG:[0-9]+]]] +; CHECK: str [[REG0]], [x0] +; CHECK: ldur [[REG1:q[0-9]+]], [x[[BASEREG]], #15] +; CHECK: stur [[REG1]], [x0, #15] tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str1, i64 0, i64 0), i64 31, i1 false) ret void } @@ -50,10 +50,10 @@ define void @t3(i8* nocapture %C) nounwind { entry: ; CHECK-LABEL: t3: -; CHECK: ldr [[DEST:q[0-9]+]], [x[[BASEREG]]] -; CHECK: ldr [[REG4:x[0-9]+]], [x[[BASEREG:[0-9]+]], #16] -; CHECK: str [[REG4]], [x0, #16] +; CHECK: ldr [[DEST:q[0-9]+]], [x[[BASEREG:[0-9]+]]] ; CHECK: str [[DEST]], [x0] +; CHECK: ldr [[REG4:x[0-9]+]], [x[[BASEREG]], #16] +; CHECK: str [[REG4]], [x0, #16] tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([24 x i8], [24 x i8]* @.str3, i64 0, i64 0), i64 24, i1 false) ret void } diff --git a/llvm/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll b/llvm/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll --- a/llvm/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll +++ b/llvm/test/CodeGen/AArch64/arm64-misaligned-memcpy-inline.ll @@ -26,14 +26,14 @@ ; loads and stores if strict-alignment is turned on. define void @t2(i8* %out, i8* %in) { ; CHECK-LABEL: t2: -; CHECK: ldrb w{{[0-9]+}}, [x1, #3] -; CHECK-NEXT: ldrb w{{[0-9]+}}, [x1, #2] -; CHECK-NEXT: ldrb w{{[0-9]+}}, [x1, #1] -; CHECK-NEXT: ldrb w{{[0-9]+}}, [x1] -; CHECK-NEXT: strb w{{[0-9]+}}, [x0, #3] -; CHECK-NEXT: strb w{{[0-9]+}}, [x0, #2] -; CHECK-NEXT: strb w{{[0-9]+}}, [x0, #1] -; CHECK-NEXT: strb w{{[0-9]+}}, [x0] +; CHECK: ldrb w[[V0:[0-9]+]], [x1] +; CHECK-NEXT: ldrb w[[V1:[0-9]+]], [x1, #1] +; CHECK-NEXT: ldrb w[[V2:[0-9]+]], [x1, #2] +; CHECK-NEXT: ldrb w[[V3:[0-9]+]], [x1, #3] +; CHECK-NEXT: strb w[[V0]], [x0] +; CHECK-NEXT: strb w[[V1]], [x0, #1] +; CHECK-NEXT: strb w[[V2]], [x0, #2] +; CHECK-NEXT: strb w[[V3]], [x0, #3] entry: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 4, i1 false) ret void diff --git a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll --- a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll +++ b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll @@ -8,8 +8,17 @@ ; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} ; MIR-LABEL: name: test_memcpy -; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) { ; CHECK-LABEL: test_memcpy: ; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] @@ -30,8 +39,17 @@ } ; MIR-LABEL: name: test_memcpy_inline -; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) define i32 @test_memcpy_inline(i32* nocapture %p, i32* nocapture readonly %q) { ; CHECK-LABEL: test_memcpy_inline: ; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] @@ -95,8 +113,17 @@ } ; MIR-LABEL: name: test_mempcpy -; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 1, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 1, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) define i32 @test_mempcpy(i32* nocapture %p, i32* nocapture readonly %q) { ; CHECK-LABEL: test_mempcpy: ; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll --- a/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll +++ b/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll @@ -12,8 +12,17 @@ ; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} ; MIR-LABEL: name: test_memcpy -; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) -; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]], addrspace 1) +; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]], addrspace 1) define i32 @test_memcpy(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { ; Check loads of %q are scheduled ahead of that store of the memcpy on %p. ; CHECK-LABEL: test_memcpy: @@ -34,8 +43,17 @@ } ; MIR-LABEL: name: test_memcpy_inline -; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) -; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]], addrspace 1) +; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]], addrspace 1) define i32 @test_memcpy_inline(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { ; Check loads of %q are scheduled ahead of that store of the memcpy on %p. ; CHECK-LABEL: test_memcpy_inline: diff --git a/llvm/test/CodeGen/PowerPC/pr45301.ll b/llvm/test/CodeGen/PowerPC/pr45301.ll --- a/llvm/test/CodeGen/PowerPC/pr45301.ll +++ b/llvm/test/CodeGen/PowerPC/pr45301.ll @@ -14,19 +14,19 @@ ; CHECK-NEXT: addis r4, r2, g@toc@ha ; CHECK-NEXT: addi r4, r4, g@toc@l ; CHECK-NEXT: ld r5, 0(r4) -; CHECK-NEXT: std r5, 0(r3) -; CHECK-NEXT: ld r5, 16(r4) -; CHECK-NEXT: std r5, 16(r3) -; CHECK-NEXT: ld r6, 8(r4) -; CHECK-NEXT: std r6, 8(r3) -; CHECK-NEXT: ld r6, 24(r4) -; CHECK-NEXT: std r6, 24(r3) -; CHECK-NEXT: lwz r6, 0(r3) +; CHECK-NEXT: ld r6, 16(r4) +; CHECK-NEXT: ld r7, 8(r4) +; CHECK-NEXT: ld r8, 24(r4) ; CHECK-NEXT: ld r4, 32(r4) +; CHECK-NEXT: std r5, 0(r3) ; CHECK-NEXT: std r4, 32(r3) ; CHECK-NEXT: li r4, 20 -; CHECK-NEXT: stwbrx r6, 0, r3 -; CHECK-NEXT: stwbrx r5, r3, r4 +; CHECK-NEXT: lwz r5, 0(r3) +; CHECK-NEXT: std r7, 8(r3) +; CHECK-NEXT: std r8, 24(r3) +; CHECK-NEXT: std r6, 16(r3) +; CHECK-NEXT: stwbrx r5, 0, r3 +; CHECK-NEXT: stwbrx r6, r3, r4 ; CHECK-NEXT: addi r1, r1, 112 ; CHECK-NEXT: ld r0, 16(r1) ; CHECK-NEXT: mtlr r0 diff --git a/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll b/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll --- a/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll +++ b/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll @@ -11,10 +11,19 @@ ; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} ; MIR-LABEL: name: test_memcpy -; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) { %p0 = bitcast i32* %p to i8* %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 @@ -28,10 +37,19 @@ } ; MIR-LABEL: name: test_memcpy_inline -; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) define i32 @test_memcpy_inline(i32* nocapture %p, i32* nocapture readonly %q) { %p0 = bitcast i32* %p to i8* %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 @@ -76,10 +94,18 @@ } ; MIR-LABEL: name: test_mempcpy -; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) -; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-DAG: ![[MMDOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"MemcpyLoweringDomain"} +; MIR-DAG: ![[MMSCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Src"} +; MIR-DAG: ![[MMSCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[MMDOMAIN]], !"Dst"} +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE0]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{![[SCOPE0]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET2:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE1]]} +; MIR-DAG: ![[MMSET3:[0-9]+]] = !{![[SCOPE1]], ![[MMSCOPE0]]} +; MIR: body: +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 1, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 1, !alias.scope ![[MMSET0]], !noalias ![[MMSET2]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 1, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 1, !alias.scope ![[MMSET1]], !noalias ![[MMSET3]]) define i32 @test_mempcpy(i32* nocapture %p, i32* nocapture readonly %q) { %p0 = bitcast i32* %p to i8* %add.ptr = getelementptr inbounds i32, i32* %p, i64 4