[MIRParser] Add machine metadata.

[hliao]

Summary

• Add standalone metadata parsing support so that machine metadata nodes could be populated before and accessed during MIR is parsed.

[hliao]

Rebase

[arsenm]

llvm/lib/CodeGen/MIRParser/MIParser.cpp

1248

Needs tests for the various parse errors

llvm/test/CodeGen/AArch64/memcpy-scoped-aa.mir

4 ↗

(On Diff #348383)

Parser/printer tests should go in test/CodeGen/MIR/*

llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.mir

221 ↗

(On Diff #348383)

Can you drop the register section and some of the other noise?

[hliao]

• Relocate tests and add extra tests for parsing errors.
• Enhance the parser to report any missing definitions on forward referenced metadata.

[hliao]

Rebase again.

[hliao]

Rebase and kindly PING for review.

[hliao]

Kindly PING after rebasing.

[hliao]

Kindly PING again.

llvm/include/llvm/CodeGen/MIRParser/MIParser.h

 #ifndef LLVM_CODEGEN_MIRPARSER_MIPARSER_H
 #define LLVM_CODEGEN_MIRPARSER_MIPARSER_H
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/SMLoc.h"
+#include <utility>
 
 namespace llvm {
 
 class MachineBasicBlock;
 class MachineFunction;
 class MDNode;
 class RegisterBank;
 struct SlotMapping;

 
 struct PerFunctionMIParsingState {
   BumpPtrAllocator Allocator;
   MachineFunction &MF;
   SourceMgr *SM;
   const SlotMapping &IRSlots;
   PerTargetMIParsingState &Target;
 
+  std::map<unsigned, TrackingMDNodeRef> MachineMetadataNodes;
+  std::map<unsigned, std::pair<TempMDTuple, SMLoc>> MachineForwardRefMDNodes;
+
   DenseMap<unsigned, MachineBasicBlock *> MBBSlots;
   DenseMap<Register, VRegInfo *> VRegInfos;
   StringMap<VRegInfo *> VRegInfosNamed;
   DenseMap<unsigned, int> FixedStackObjectSlots;
   DenseMap<unsigned, int> StackObjectSlots;
   DenseMap<unsigned, unsigned> ConstantPoolSlots;
   DenseMap<unsigned, unsigned> JumpTableSlots;
 
bool parseVirtualRegisterReference(PerFunctionMIParsingState &PFS,
                                    SMDiagnostic &Error);
 
 bool parseStackObjectReference(PerFunctionMIParsingState &PFS, int &FI,
                                StringRef Src, SMDiagnostic &Error);
 
 bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node, StringRef Src,
                  SMDiagnostic &Error);
 
+bool parseMachineMetadata(PerFunctionMIParsingState &PFS, StringRef Src,
+                          SMRange SourceRange, SMDiagnostic &Error);
+
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_MIRPARSER_MIPARSER_H

llvm/lib/CodeGen/MIRParser/MILexer.h

enum TokenKind {
     kw_shufflemask,
     kw_pre_instr_symbol,
     kw_post_instr_symbol,
     kw_heap_alloc_marker,
     kw_bbsections,
     kw_unknown_size,
     kw_unknown_address,
 
+    // Metadata types.
+    kw_distinct,
+
     // Named metadata keywords
     md_tbaa,
     md_alias_scope,
     md_noalias,
     md_range,
     md_diexpr,
     md_dilocation,
 

llvm/lib/CodeGen/MIRParser/MILexer.cpp

return StringSwitch<MIToken::TokenKind>(Identifier)
       .Case("intpred", MIToken::kw_intpred)
       .Case("shufflemask", MIToken::kw_shufflemask)
       .Case("pre-instr-symbol", MIToken::kw_pre_instr_symbol)
       .Case("post-instr-symbol", MIToken::kw_post_instr_symbol)
       .Case("heap-alloc-marker", MIToken::kw_heap_alloc_marker)
       .Case("bbsections", MIToken::kw_bbsections)
       .Case("unknown-size", MIToken::kw_unknown_size)
       .Case("unknown-address", MIToken::kw_unknown_address)
+      .Case("distinct", MIToken::kw_distinct)
       .Default(MIToken::Identifier);
 }
 
 static Cursor maybeLexIdentifier(Cursor C, MIToken &Token) {
   if (!isalpha(C.peek()) && C.peek() != '_')
     return None;
   auto Range = C;
   while (isIdentifierChar(C.peek()))

llvm/lib/CodeGen/MIRParser/MIParser.cpp

if (TiedDefIdx)
              "Only used register operands can be tied");
   }
 };
 
 class MIParser {
   MachineFunction &MF;
   SMDiagnostic &Error;
   StringRef Source, CurrentSource;
+  SMRange SourceRange;
   MIToken Token;
   PerFunctionMIParsingState &PFS;
   /// Maps from slot numbers to function's unnamed basic blocks.
   DenseMap<unsigned, const BasicBlock *> Slots2BasicBlocks;
 
 public:
   MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
            StringRef Source);
+  MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
+           StringRef Source, SMRange SourceRange);
 
   /// \p SkipChar gives the number of characters to skip before looking
   /// for the next token.
   void lex(unsigned SkipChar = 0);
 
   /// Report an error at the current location with the given message.
   ///
   /// This function always return true.
public:
   bool parseBasicBlocks();
   bool parse(MachineInstr *&MI);
   bool parseStandaloneMBB(MachineBasicBlock *&MBB);
   bool parseStandaloneNamedRegister(Register &Reg);
   bool parseStandaloneVirtualRegister(VRegInfo *&Info);
   bool parseStandaloneRegister(Register &Reg);
   bool parseStandaloneStackObject(int &FI);
   bool parseStandaloneMDNode(MDNode *&Node);
+  bool parseMachineMetadata();
+  bool parseMDTuple(MDNode *&MD, bool IsDistinct);
+  bool parseMDNodeVector(SmallVectorImpl<Metadata *> &Elts);
+  bool parseMetadata(Metadata *&MD);
 
   bool
   parseBasicBlockDefinition(DenseMap<unsigned, MachineBasicBlock *> &MBBSlots);
   bool parseBasicBlock(MachineBasicBlock &MBB,
                        MachineBasicBlock *&AddFalthroughFrom);
   bool parseBasicBlockLiveins(MachineBasicBlock &MBB);
   bool parseBasicBlockSuccessors(MachineBasicBlock &MBB);
 
private:
   const BasicBlock *getIRBlock(unsigned Slot, const Function &F);
 
   /// Get or create an MCSymbol for a given name.
   MCSymbol *getOrCreateMCSymbol(StringRef Name);
 
   /// parseStringConstant
   ///   ::= StringConstant
   bool parseStringConstant(std::string &Result);
+
+  /// Map the location in the MI string to the corresponding location specified
+  /// in `SourceRange`.
+  SMLoc mapSMLoc(StringRef::iterator Loc);
 };
 
 } // end anonymous namespace
 
 MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
                    StringRef Source)
     : MF(PFS.MF), Error(Error), Source(Source), CurrentSource(Source), PFS(PFS)
 {}
 
+MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
+                   StringRef Source, SMRange SourceRange)
+    : MF(PFS.MF), Error(Error), Source(Source), CurrentSource(Source),
+      SourceRange(SourceRange), PFS(PFS) {}
+
 void MIParser::lex(unsigned SkipChar) {
   CurrentSource = lexMIToken(
       CurrentSource.slice(SkipChar, StringRef::npos), Token,
       [this](StringRef::iterator Loc, const Twine &Msg) { error(Loc, Msg); });
 }
 
 bool MIParser::error(const Twine &Msg) { return error(Token.location(), Msg); }
 
bool MIParser::error(StringRef::iterator Loc, const Twine &Msg) {
   }
   // Create a diagnostic for a YAML string literal.
   Error = SMDiagnostic(SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
                        Loc - Source.data(), SourceMgr::DK_Error, Msg.str(),
                        Source, None, None);
   return true;
 }
 
+SMLoc MIParser::mapSMLoc(StringRef::iterator Loc) {
+  assert(SourceRange.isValid() && "Invalid source range");
+  assert(Loc >= Source.data() && Loc <= (Source.data() + Source.size()));
+  return SMLoc::getFromPointer(SourceRange.Start.getPointer() +
+                               (Loc - Source.data()));
+}
+
 typedef function_ref<bool(StringRef::iterator Loc, const Twine &)>
     ErrorCallbackType;
 
 static const char *toString(MIToken::TokenKind TokenKind) {
   switch (TokenKind) {
   case MIToken::comma:
     return "','";
   case MIToken::equal:
if (parseDILocation(Node))
       return true;
   } else
     return error("expected a metadata node");
   if (Token.isNot(MIToken::Eof))
     return error("expected end of string after the metadata node");
   return false;
 }
 
+bool MIParser::parseMachineMetadata() {
+  lex();
+  if (Token.isNot(MIToken::exclaim))
+    return error("expected a metadata node");
+
+  lex();
+  if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
+    return error("expected metadata id after '!'");
+  unsigned ID = 0;
+  if (getUnsigned(ID))
+    return true;
+  lex();
+  if (expectAndConsume(MIToken::equal))
+    return true;
+  bool IsDistinct = Token.is(MIToken::kw_distinct);
+  if (IsDistinct)
+    lex();
+  if (Token.isNot(MIToken::exclaim))
+    return error("expected a metadata node");
+  lex();
+
+  MDNode *MD;
+  if (parseMDTuple(MD, IsDistinct))
+    return true;
+
+  auto FI = PFS.MachineForwardRefMDNodes.find(ID);
+  if (FI != PFS.MachineForwardRefMDNodes.end()) {
+    FI->second.first->replaceAllUsesWith(MD);
+    PFS.MachineForwardRefMDNodes.erase(FI);
+
+    assert(PFS.MachineMetadataNodes[ID] == MD && "Tracking VH didn't work");
+  } else {
+    if (PFS.MachineMetadataNodes.count(ID))
+      return error("Metadata id is already used");
+    PFS.MachineMetadataNodes[ID].reset(MD);
+  }
+
+  return false;
+}
+
+bool MIParser::parseMDTuple(MDNode *&MD, bool IsDistinct) {
+  SmallVector<Metadata *, 16> Elts;
+  if (parseMDNodeVector(Elts))
+    return true;
+  MD = (IsDistinct ? MDTuple::getDistinct
+                   : MDTuple::get)(MF.getFunction().getContext(), Elts);
+  return false;
+}
+
+bool MIParser::parseMDNodeVector(SmallVectorImpl<Metadata *> &Elts) {
+  if (Token.isNot(MIToken::lbrace))
+    return error("expected '{' here");
+  lex();
+
+  if (Token.is(MIToken::rbrace)) {
+    lex();
+    return false;
+  }
+
+  do {
+    Metadata *MD;
+    if (parseMetadata(MD))
+      return true;
+
+    Elts.push_back(MD);
+
+    if (Token.isNot(MIToken::comma))
+      break;
+    lex();
+  } while (true);
+
+  if (Token.isNot(MIToken::rbrace))
+    return error("expected end of metadata node");
+  lex();
+
+  return false;
+}
+
+// ::= !42
+// ::= !"string"
+bool MIParser::parseMetadata(Metadata *&MD) {
+  if (Token.isNot(MIToken::exclaim))
+    return error("expected '!' here");
+  lex();
+
+  if (Token.is(MIToken::StringConstant)) {
+    std::string Str;
+    if (parseStringConstant(Str))
+      return true;
+    MD = MDString::get(MF.getFunction().getContext(), Str);
+    return false;
+  }
+
+  if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
+    return error("expected metadata id after '!'");
+
+  SMLoc Loc = mapSMLoc(Token.location());
+
+  unsigned ID = 0;
+  if (getUnsigned(ID))
+    return true;
+  lex();
+
+  auto NodeInfo = PFS.IRSlots.MetadataNodes.find(ID);
+  if (NodeInfo != PFS.IRSlots.MetadataNodes.end()) {
+    MD = NodeInfo->second.get();
+    return false;
+  }
+  // Check machine metadata.
+  NodeInfo = PFS.MachineMetadataNodes.find(ID);
+  if (NodeInfo != PFS.MachineMetadataNodes.end()) {
+    MD = NodeInfo->second.get();
+    return false;
+  }
+  // Forward reference.
+  auto &FwdRef = PFS.MachineForwardRefMDNodes[ID];
+  FwdRef = std::make_pair(
+      MDTuple::getTemporary(MF.getFunction().getContext(), None), Loc);
+  PFS.MachineMetadataNodes[ID].reset(FwdRef.first.get());
+  MD = FwdRef.first.get();
+
+  return false;
+}
+
 static const char *printImplicitRegisterFlag(const MachineOperand &MO) {
   assert(MO.isImplicit());
   return MO.isDef() ? "implicit-def" : "implicit";
 }
 
 static std::string getRegisterName(const TargetRegisterInfo *TRI,
                                    Register Reg) {
   assert(Register::isPhysicalRegister(Reg) && "expected phys reg");
bool MIParser::parseMDNode(MDNode *&Node) {
   auto Loc = Token.location();
   lex();
   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
     return error("expected metadata id after '!'");
   unsigned ID;
   if (getUnsigned(ID))
     return true;
   auto NodeInfo = PFS.IRSlots.MetadataNodes.find(ID);
-  if (NodeInfo == PFS.IRSlots.MetadataNodes.end())
+  if (NodeInfo == PFS.IRSlots.MetadataNodes.end()) {
+    NodeInfo = PFS.MachineMetadataNodes.find(ID);
+    if (NodeInfo == PFS.MachineMetadataNodes.end())
       return error(Loc, "use of undefined metadata '!" + Twine(ID) + "'");
+  }
   lex();
   Node = NodeInfo->second.get();
   return false;
 }
 
 bool MIParser::parseDIExpression(MDNode *&Expr) {
   assert(Token.is(MIToken::md_diexpr));
   lex();
bool llvm::parseStackObjectReference(PerFunctionMIParsingState &PFS,
   return MIParser(PFS, Error, Src).parseStandaloneStackObject(FI);
 }
 
 bool llvm::parseMDNode(PerFunctionMIParsingState &PFS,
                        MDNode *&Node, StringRef Src, SMDiagnostic &Error) {
   return MIParser(PFS, Error, Src).parseStandaloneMDNode(Node);
 }
 
+bool llvm::parseMachineMetadata(PerFunctionMIParsingState &PFS, StringRef Src,
+                                SMRange SrcRange, SMDiagnostic &Error) {
+  return MIParser(PFS, Error, Src, SrcRange).parseMachineMetadata();
+}
+
 bool MIRFormatter::parseIRValue(StringRef Src, MachineFunction &MF,
                                 PerFunctionMIParsingState &PFS, const Value *&V,
                                 ErrorCallbackType ErrorCallback) {
   MIToken Token;
   Src = lexMIToken(Src, Token, [&](StringRef::iterator Loc, const Twine &Msg) {
     ErrorCallback(Loc, Msg);
   });
   V = nullptr;
 
   return ::parseIRValue(Token, PFS, V, ErrorCallback);
 }

llvm/lib/CodeGen/MIRParser/MIRParser.cpp

public:
 
   bool initializeConstantPool(PerFunctionMIParsingState &PFS,
                               MachineConstantPool &ConstantPool,
                               const yaml::MachineFunction &YamlMF);
 
   bool initializeJumpTableInfo(PerFunctionMIParsingState &PFS,
                                const yaml::MachineJumpTable &YamlJTI);
 
+  bool parseMachineMetadataNodes(PerFunctionMIParsingState &PFS,
+                                 MachineFunction &MF,
+                                 const yaml::MachineFunction &YMF);
+
 private:
   bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node,
                    const yaml::StringValue &Source);
 
   bool parseMBBReference(PerFunctionMIParsingState &PFS,
                          MachineBasicBlock *&MBB,
                          const yaml::StringValue &Source);
 
+  bool parseMachineMetadata(PerFunctionMIParsingState &PFS,
+                            const yaml::StringValue &Source);
+
   /// Return a MIR diagnostic converted from an MI string diagnostic.
   SMDiagnostic diagFromMIStringDiag(const SMDiagnostic &Error,
                                     SMRange SourceRange);
 
   /// Return a MIR diagnostic converted from a diagnostic located in a YAML
   /// block scalar string.
   SMDiagnostic diagFromBlockStringDiag(const SMDiagnostic &Error,
                                        SMRange SourceRange);
MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF,
   if (parseRegisterInfo(PFS, YamlMF))
     return true;
   if (!YamlMF.Constants.empty()) {
     auto *ConstantPool = MF.getConstantPool();
     assert(ConstantPool && "Constant pool must be created");
     if (initializeConstantPool(PFS, *ConstantPool, YamlMF))
       return true;
   }
+  if (!YamlMF.MachineMetadataNodes.empty() &&
+      parseMachineMetadataNodes(PFS, MF, YamlMF))
+    return true;
 
   StringRef BlockStr = YamlMF.Body.Value.Value;
   SMDiagnostic Error;
   SourceMgr BlockSM;
   BlockSM.AddNewSourceBuffer(
       MemoryBuffer::getMemBuffer(BlockStr, "",/*RequiresNullTerminator=*/false),
       SMLoc());
   PFS.SM = &BlockSM;
bool MIRParserImpl::parseMBBReference(PerFunctionMIParsingState &PFS,
                                       MachineBasicBlock *&MBB,
                                       const yaml::StringValue &Source) {
   SMDiagnostic Error;
   if (llvm::parseMBBReference(PFS, MBB, Source.Value, Error))
     return error(Error, Source.SourceRange);
   return false;
 }
 
+bool MIRParserImpl::parseMachineMetadata(PerFunctionMIParsingState &PFS,
+                                         const yaml::StringValue &Source) {
+  SMDiagnostic Error;
+  if (llvm::parseMachineMetadata(PFS, Source.Value, Source.SourceRange, Error))
+    return error(Error, Source.SourceRange);
+  return false;
+}
+
+bool MIRParserImpl::parseMachineMetadataNodes(
+    PerFunctionMIParsingState &PFS, MachineFunction &MF,
+    const yaml::MachineFunction &YMF) {
+  for (auto &MDS : YMF.MachineMetadataNodes) {
+    if (parseMachineMetadata(PFS, MDS))
+      return true;
+  }
+  // Report missing definitions from forward referenced nodes.
+  if (!PFS.MachineForwardRefMDNodes.empty())
+    return error(PFS.MachineForwardRefMDNodes.begin()->second.second,
+                 "use of undefined metadata '!" +
+                     Twine(PFS.MachineForwardRefMDNodes.begin()->first) + "'");
+  return false;
+}
+
 SMDiagnostic MIRParserImpl::diagFromMIStringDiag(const SMDiagnostic &Error,
                                                  SMRange SourceRange) {
   assert(SourceRange.isValid() && "Invalid source range");
   SMLoc Loc = SourceRange.Start;
   bool HasQuote = Loc.getPointer() < SourceRange.End.getPointer() &&
                   *Loc.getPointer() == '\'';
   // Translate the location of the error from the location in the MI string to
   // the corresponding location in the MIR file.

llvm/test/CodeGen/MIR/AArch64/machine-metadata-error.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+--- |
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-unknown-linux-gnu"
+
+  define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) {
+    ret i32 0
+  }
+...
+---
+name:            test_memcpy
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!10 = !{!9}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!5 = !{!6}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+body:             |
+  bb.0 (%ir-block.0):
+  ; CHECK: [[@LINE+1]]:78: use of undefined metadata '!11'
+    %2:fpr128 = LDRQui %0, 1 :: (load 16, align 4, !alias.scope !5, !noalias !11)
+
+...

llvm/test/CodeGen/MIR/AArch64/machine-metadata.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-linux-gnu -run-pass=none -o - %s | FileCheck %s
+--- |
+  ; ModuleID = 'test/CodeGen/AArch64/memcpy-scoped-aa.ll'
+  source_filename = "test/CodeGen/AArch64/memcpy-scoped-aa.ll"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-unknown-linux-gnu"
+
+  define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) {
+    %p0 = bitcast i32* %p to i8*
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 4
+    %p1 = bitcast i32* %add.ptr to i8*
+    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !0, !noalias !3
+    %v0 = load i32, i32* %q, align 4, !alias.scope !3, !noalias !0
+    %q1 = getelementptr inbounds i32, i32* %q, i64 1
+    %v1 = load i32, i32* %q1, align 4, !alias.scope !3, !noalias !0
+    %add = add i32 %v0, %v1
+    ret i32 %add
+  }
+
+  define i32 @test_memcpy_inline(i32* nocapture %p, i32* nocapture readonly %q) {
+    %p0 = bitcast i32* %p to i8*
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 4
+    %p1 = bitcast i32* %add.ptr to i8*
+    tail call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !0, !noalias !3
+    %v0 = load i32, i32* %q, align 4, !alias.scope !3, !noalias !0
+    %q1 = getelementptr inbounds i32, i32* %q, i64 1
+    %v1 = load i32, i32* %q1, align 4, !alias.scope !3, !noalias !0
+    %add = add i32 %v0, %v1
+    ret i32 %add
+  }
+
+  define i32 @test_mempcpy(i32* nocapture %p, i32* nocapture readonly %q) {
+    %p0 = bitcast i32* %p to i8*
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 4
+    %p1 = bitcast i32* %add.ptr to i8*
+    %call = tail call i8* @mempcpy(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16), !alias.scope !0, !noalias !3
+    %v0 = load i32, i32* %q, align 4, !alias.scope !3, !noalias !0
+    %q1 = getelementptr inbounds i32, i32* %q, i64 1
+    %v1 = load i32, i32* %q1, align 4, !alias.scope !3, !noalias !0
+    %add = add i32 %v0, %v1
+    ret i32 %add
+  }
+
+  ; Function Attrs: argmemonly nofree nounwind willreturn
+  declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) #0
+
+  ; Function Attrs: argmemonly nofree nounwind willreturn
+  declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64 immarg, i1 immarg) #0
+
+  declare i8* @mempcpy(i8*, i8*, i64)
+
+  attributes #0 = { argmemonly nofree nounwind willreturn }
+
+  !0 = !{!1}
+  !1 = distinct !{!1, !2, !"bax: %p"}
+  !2 = distinct !{!2, !"bax"}
+  !3 = !{!4}
+  !4 = distinct !{!4, !2, !"bax: %q"}
+
+...
+---
+name:            test_memcpy
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!10 = !{!1, !9}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!5 = !{!1, !6}'
+  - '!11 = !{!4, !6}'
+  - '!8 = !{!4, !9}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_memcpy
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[COPY1]], 1 :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    ; CHECK: STRQui killed [[LDRQui]], [[COPY1]], 0 :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    ; CHECK: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    ; CHECK: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    ; CHECK: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[LDRWui]], killed [[LDRWui1]]
+    ; CHECK: $w0 = COPY [[ADDWrr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %1:gpr64common = COPY $x1
+    %0:gpr64common = COPY $x0
+    %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    %3:gpr32 = LDRWui %1, 0 :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    %4:gpr32 = LDRWui %1, 1 :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    %5:gpr32 = ADDWrr killed %3, killed %4
+    $w0 = COPY %5
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            test_memcpy_inline
+machineMetadataNodes:
+  - '!10 = !{!1, !9}'
+  - '!5 = !{!1, !6}'
+  - '!8 = !{!4, !9}'
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!11 = !{!4, !6}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_memcpy_inline
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[COPY1]], 1 :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    ; CHECK: STRQui killed [[LDRQui]], [[COPY1]], 0 :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    ; CHECK: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    ; CHECK: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    ; CHECK: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[LDRWui]], killed [[LDRWui1]]
+    ; CHECK: $w0 = COPY [[ADDWrr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %1:gpr64common = COPY $x1
+    %0:gpr64common = COPY $x0
+    %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    %3:gpr32 = LDRWui %1, 0 :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    %4:gpr32 = LDRWui %1, 1 :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    %5:gpr32 = ADDWrr killed %3, killed %4
+    $w0 = COPY %5
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            test_mempcpy
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!8 = !{!4, !9}'
+  - '!5 = !{!1, !6}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!11 = !{!4, !6}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!10 = !{!1, !9}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_mempcpy
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK: [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[COPY1]], 1 :: (load 16 from %ir.p1, align 1, !alias.scope !5, !noalias !8)
+    ; CHECK: STRQui killed [[LDRQui]], [[COPY1]], 0 :: (store 16 into %ir.p0, align 1, !alias.scope !10, !noalias !11)
+    ; CHECK: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    ; CHECK: [[LDRWui1:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 1 :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    ; CHECK: [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[LDRWui]], killed [[LDRWui1]]
+    ; CHECK: $w0 = COPY [[ADDWrr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %1:gpr64common = COPY $x1
+    %0:gpr64common = COPY $x0
+    %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 1, !alias.scope !5, !noalias !8)
+    STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 1, !alias.scope !10, !noalias !11)
+    %3:gpr32 = LDRWui %1, 0 :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    %4:gpr32 = LDRWui %1, 1 :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    %5:gpr32 = ADDWrr killed %3, killed %4
+    $w0 = COPY %5
+    RET_ReallyLR implicit $w0
+
+...

llvm/test/CodeGen/MIR/AMDGPU/machine-metadata-error.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+--- |
+  target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+  target triple = "amdgcn-amd-amdhsa"
+
+  define i32 @test_memcpy(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) #0 {
+    ret i32 0
+  }
+...
+---
+name:            test_memcpy
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!5 = !{!6}'
+  - '!10 = !{!9}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+body:             |
+  bb.0 (%ir-block.0):
+  ; CHECK: [[@LINE+1]]:113: use of undefined metadata '!11'
+    %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16, align 4, !alias.scope !5, !noalias !11, addrspace 1)
+
+...

llvm/test/CodeGen/MIR/AMDGPU/machine-metadata.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -run-pass=none -o - %s | FileCheck %s
+--- |
+  ; ModuleID = 'test/CodeGen/AMDGPU/memcpy-scoped-aa.ll'
+  source_filename = "test/CodeGen/AMDGPU/memcpy-scoped-aa.ll"
+  target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+  target triple = "amdgcn-amd-amdhsa"
+
+  define i32 @test_memcpy(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) #0 {
+    %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)*
+    %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4
+    %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)*
+    tail call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !0, !noalias !3
+    %1 = bitcast i32 addrspace(1)* %q to <2 x i32> addrspace(1)*
+    %2 = load <2 x i32>, <2 x i32> addrspace(1)* %1, align 4, !alias.scope !3, !noalias !0
+    %v01 = extractelement <2 x i32> %2, i32 0
+    %v12 = extractelement <2 x i32> %2, i32 1
+    %add = add i32 %v01, %v12
+    ret i32 %add
+  }
+
+  define i32 @test_memcpy_inline(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) #0 {
+    %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)*
+    %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4
+    %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)*
+    tail call void @llvm.memcpy.inline.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !0, !noalias !3
+    %1 = bitcast i32 addrspace(1)* %q to <2 x i32> addrspace(1)*
+    %2 = load <2 x i32>, <2 x i32> addrspace(1)* %1, align 4, !alias.scope !3, !noalias !0
+    %v01 = extractelement <2 x i32> %2, i32 0
+    %v12 = extractelement <2 x i32> %2, i32 1
+    %add = add i32 %v01, %v12
+    ret i32 %add
+  }
+
+  ; Function Attrs: argmemonly nofree nounwind willreturn
+  declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* noalias nocapture writeonly, i8 addrspace(1)* noalias nocapture readonly, i64, i1 immarg) #1
+
+  ; Function Attrs: argmemonly nofree nounwind willreturn
+  declare void @llvm.memcpy.inline.p1i8.p1i8.i64(i8 addrspace(1)* noalias nocapture writeonly, i8 addrspace(1)* noalias nocapture readonly, i64 immarg, i1 immarg) #1
+
+  ; Function Attrs: convergent nounwind willreturn
+  declare { i1, i32 } @llvm.amdgcn.if.i32(i1) #2
+
+  ; Function Attrs: convergent nounwind willreturn
+  declare { i1, i32 } @llvm.amdgcn.else.i32.i32(i32) #2
+
+  ; Function Attrs: convergent nounwind readnone willreturn
+  declare i32 @llvm.amdgcn.if.break.i32(i1, i32) #3
+
+  ; Function Attrs: convergent nounwind willreturn
+  declare i1 @llvm.amdgcn.loop.i32(i32) #2
+
+  ; Function Attrs: convergent nounwind willreturn
+  declare void @llvm.amdgcn.end.cf.i32(i32) #2
+
+  attributes #0 = { "target-cpu"="gfx1010" }
+  attributes #1 = { argmemonly nofree nounwind willreturn "target-cpu"="gfx1010" }
+  attributes #2 = { convergent nounwind willreturn }
+  attributes #3 = { convergent nounwind readnone willreturn }
+
+  !0 = !{!1}
+  !1 = distinct !{!1, !2, !"bax: %p"}
+  !2 = distinct !{!2, !"bax"}
+  !3 = !{!4}
+  !4 = distinct !{!4, !2, !"bax: %q"}
+
+...
+---
+name:            test_memcpy
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!11 = !{!4, !6}'
+  - '!5 = !{!1, !6}'
+  - '!8 = !{!4, !9}'
+  - '!10 = !{!1, !9}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+
+    ; CHECK-LABEL: name: test_memcpy
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+    ; CHECK: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+    ; CHECK: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+    ; CHECK: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY5]], 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8, addrspace 1)
+    ; CHECK: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+    ; CHECK: GLOBAL_STORE_DWORDX4 [[COPY6]], killed [[GLOBAL_LOAD_DWORDX4_]], 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11, addrspace 1)
+    ; CHECK: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+    ; CHECK: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 killed [[COPY7]], 0, 0, implicit $exec :: (load 8 from %ir.1, align 4, !alias.scope !3, !noalias !0, addrspace 1)
+    ; CHECK: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX2_]].sub0
+    ; CHECK: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX2_]].sub1
+    ; CHECK: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 killed [[COPY8]], killed [[COPY9]], 0, implicit $exec
+    ; CHECK: [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+    ; CHECK: [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY10]]
+    ; CHECK: S_SETPC_B64_return [[COPY11]], implicit $vgpr0
+    %4:sreg_64 = COPY $sgpr30_sgpr31
+    %3:vgpr_32 = COPY $vgpr3
+    %2:vgpr_32 = COPY $vgpr2
+    %1:vgpr_32 = COPY $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %17:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
+    %18:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+    %9:vreg_64 = COPY %18
+    %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8, addrspace 1)
+    %10:vreg_64 = COPY %18
+    GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11, addrspace 1)
+    %12:vreg_64 = COPY %17
+    %11:vreg_64 = GLOBAL_LOAD_DWORDX2 killed %12, 0, 0, implicit $exec :: (load 8 from %ir.1, align 4, !alias.scope !3, !noalias !0, addrspace 1)
+    %13:vgpr_32 = COPY %11.sub0
+    %14:vgpr_32 = COPY %11.sub1
+    %15:vgpr_32 = V_ADD_U32_e64 killed %13, killed %14, 0, implicit $exec
+    %5:ccr_sgpr_64 = COPY %4
+    $vgpr0 = COPY %15
+    %16:ccr_sgpr_64 = COPY %5
+    S_SETPC_B64_return %16, implicit $vgpr0
+
+...
+---
+name:            test_memcpy_inline
+machineMetadataNodes:
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!11 = !{!4, !6}'
+  - '!5 = !{!1, !6}'
+  - '!8 = !{!4, !9}'
+  - '!10 = !{!1, !9}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr30_sgpr31
+
+    ; CHECK-LABEL: name: test_memcpy_inline
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+    ; CHECK: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+    ; CHECK: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+    ; CHECK: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY5]], 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8, addrspace 1)
+    ; CHECK: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+    ; CHECK: GLOBAL_STORE_DWORDX4 [[COPY6]], killed [[GLOBAL_LOAD_DWORDX4_]], 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11, addrspace 1)
+    ; CHECK: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+    ; CHECK: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 killed [[COPY7]], 0, 0, implicit $exec :: (load 8 from %ir.1, align 4, !alias.scope !3, !noalias !0, addrspace 1)
+    ; CHECK: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX2_]].sub0
+    ; CHECK: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX2_]].sub1
+    ; CHECK: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 killed [[COPY8]], killed [[COPY9]], 0, implicit $exec
+    ; CHECK: [[COPY10:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[V_ADD_U32_e64_]]
+    ; CHECK: [[COPY11:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY10]]
+    ; CHECK: S_SETPC_B64_return [[COPY11]], implicit $vgpr0
+    %4:sreg_64 = COPY $sgpr30_sgpr31
+    %3:vgpr_32 = COPY $vgpr3
+    %2:vgpr_32 = COPY $vgpr2
+    %1:vgpr_32 = COPY $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %17:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
+    %18:vreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
+    %9:vreg_64 = COPY %18
+    %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope !5, !noalias !8, addrspace 1)
+    %10:vreg_64 = COPY %18
+    GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope !10, !noalias !11, addrspace 1)
+    %12:vreg_64 = COPY %17
+    %11:vreg_64 = GLOBAL_LOAD_DWORDX2 killed %12, 0, 0, implicit $exec :: (load 8 from %ir.1, align 4, !alias.scope !3, !noalias !0, addrspace 1)
+    %13:vgpr_32 = COPY %11.sub0
+    %14:vgpr_32 = COPY %11.sub1
+    %15:vgpr_32 = V_ADD_U32_e64 killed %13, killed %14, 0, implicit $exec
+    %5:ccr_sgpr_64 = COPY %4
+    $vgpr0 = COPY %15
+    %16:ccr_sgpr_64 = COPY %5
+    S_SETPC_B64_return %16, implicit $vgpr0
+
+...

llvm/test/CodeGen/MIR/Generic/machine-metadata-err0.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '9 = distinct !{!9, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:6: expected a metadata node

llvm/test/CodeGen/MIR/Generic/machine-metadata-err1.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '! = distinct !{!9, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:8: expected metadata id after '!'

llvm/test/CodeGen/MIR/Generic/machine-metadata-err2.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct {!9, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:20: expected a metadata node

llvm/test/CodeGen/MIR/Generic/machine-metadata-err3.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!9 = distinct !{!9, !7, !"Src"}'
+...
+# CHECK: [[@LINE-2]]:37: Metadata id is already used

llvm/test/CodeGen/MIR/Generic/machine-metadata-err4.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct !!9, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:21: expected '{' here

llvm/test/CodeGen/MIR/Generic/machine-metadata-err5.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"'
+...
+# CHECK: [[@LINE-2]]:36: expected end of metadata node

llvm/test/CodeGen/MIR/Generic/machine-metadata-err6.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct !{9, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:22: expected '!' here

llvm/test/CodeGen/MIR/Generic/machine-metadata-err7.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct !{!, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:23: expected metadata id after '!'

llvm/test/CodeGen/MIR/Generic/machine-metadata-err8.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+
+--- |
+  define i32 @t0() {
+    ret i32 0
+  }
+...
+---
+name: t0
+machineMetadataNodes:
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+...
+# CHECK: [[@LINE-2]]:26: use of undefined metadata '!7'

llvm/test/CodeGen/MIR/X86/machine-metadata-error.mir

+# RUN: not llc -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# This test ensures that the MIR parser detects errors when parsing machine
+# metadata.
+--- |
+  target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+  target triple = "x86_64-unknown-linux-gnu"
+
+  define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) {
+    ret i32 0
+  }
+...
+---
+name:            test_memcpy
+machineMetadataNodes:
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!5 = !{!6}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!10 = !{!9}'
+body:             |
+  bb.0 (%ir-block.0):
+  ; CHECK: [[@LINE+1]]:96: use of undefined metadata '!11'
+    %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8, align 4, !alias.scope !5, !noalias !11)
+
+...

llvm/test/CodeGen/MIR/X86/machine-metadata.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=none -o - %s | FileCheck %s
+--- |
+  ; ModuleID = 'test/CodeGen/X86/memcpy-scoped-aa.ll'
+  source_filename = "test/CodeGen/X86/memcpy-scoped-aa.ll"
+  target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+  target triple = "x86_64-unknown-linux-gnu"
+
+  define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) {
+    %p0 = bitcast i32* %p to i8*
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 4
+    %p1 = bitcast i32* %add.ptr to i8*
+    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !0, !noalias !3
+    %v0 = load i32, i32* %q, align 4, !alias.scope !3, !noalias !0
+    %q1 = getelementptr inbounds i32, i32* %q, i64 1
+    %v1 = load i32, i32* %q1, align 4, !alias.scope !3, !noalias !0
+    %add = add i32 %v0, %v1
+    ret i32 %add
+  }
+
+  define i32 @test_memcpy_inline(i32* nocapture %p, i32* nocapture readonly %q) {
+    %p0 = bitcast i32* %p to i8*
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 4
+    %p1 = bitcast i32* %add.ptr to i8*
+    tail call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !0, !noalias !3
+    %v0 = load i32, i32* %q, align 4, !alias.scope !3, !noalias !0
+    %q1 = getelementptr inbounds i32, i32* %q, i64 1
+    %v1 = load i32, i32* %q1, align 4, !alias.scope !3, !noalias !0
+    %add = add i32 %v0, %v1
+    ret i32 %add
+  }
+
+  define i32 @test_mempcpy(i32* nocapture %p, i32* nocapture readonly %q) {
+    %p0 = bitcast i32* %p to i8*
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 4
+    %p1 = bitcast i32* %add.ptr to i8*
+    %call = tail call i8* @mempcpy(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16), !alias.scope !0, !noalias !3
+    %v0 = load i32, i32* %q, align 4, !alias.scope !3, !noalias !0
+    %q1 = getelementptr inbounds i32, i32* %q, i64 1
+    %v1 = load i32, i32* %q1, align 4, !alias.scope !3, !noalias !0
+    %add = add i32 %v0, %v1
+    ret i32 %add
+  }
+
+  ; Function Attrs: argmemonly nofree nounwind willreturn
+  declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) #0
+
+  ; Function Attrs: argmemonly nofree nounwind willreturn
+  declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64 immarg, i1 immarg) #0
+
+  declare i8* @mempcpy(i8*, i8*, i64)
+
+  attributes #0 = { argmemonly nofree nounwind willreturn }
+
+  !0 = !{!1}
+  !1 = distinct !{!1, !2, !"bax: %p"}
+  !2 = distinct !{!2, !"bax"}
+  !3 = !{!4}
+  !4 = distinct !{!4, !2, !"bax: %q"}
+
+...
+---
+name:            test_memcpy
+machineMetadataNodes:
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!8 = !{!4, !9}'
+  - '!5 = !{!1, !6}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!11 = !{!4, !6}'
+  - '!10 = !{!1, !9}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $rdi, $rsi
+
+    ; CHECK-LABEL: name: test_memcpy
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+    ; CHECK: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm [[COPY1]], 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    ; CHECK: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm [[COPY1]], 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope !5, !noalias !8)
+    ; CHECK: MOV64mr [[COPY1]], 1, $noreg, 8, $noreg, killed [[MOV64rm1]] :: (store 8 into %ir.p0 + 8, align 4, !alias.scope !10, !noalias !11)
+    ; CHECK: MOV64mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV64rm]] :: (store 8 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    ; CHECK: [[ADD32rm:%[0-9]+]]:gr32 = ADD32rm [[MOV32rm]], [[COPY]], 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    ; CHECK: $eax = COPY [[ADD32rm]]
+    ; CHECK: RET 0, $eax
+    %1:gr64 = COPY $rsi
+    %0:gr64 = COPY $rdi
+    %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope !5, !noalias !8)
+    MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope !10, !noalias !11)
+    MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    %4:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    %5:gr32 = ADD32rm %4, %1, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    $eax = COPY %5
+    RET 0, $eax
+
+...
+---
+name:            test_memcpy_inline
+machineMetadataNodes:
+  - '!8 = !{!4, !9}'
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+  - '!5 = !{!1, !6}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!11 = !{!4, !6}'
+  - '!10 = !{!1, !9}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $rdi, $rsi
+
+    ; CHECK-LABEL: name: test_memcpy_inline
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+    ; CHECK: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm [[COPY1]], 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    ; CHECK: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm [[COPY1]], 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope !5, !noalias !8)
+    ; CHECK: MOV64mr [[COPY1]], 1, $noreg, 8, $noreg, killed [[MOV64rm1]] :: (store 8 into %ir.p0 + 8, align 4, !alias.scope !10, !noalias !11)
+    ; CHECK: MOV64mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV64rm]] :: (store 8 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    ; CHECK: [[ADD32rm:%[0-9]+]]:gr32 = ADD32rm [[MOV32rm]], [[COPY]], 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    ; CHECK: $eax = COPY [[ADD32rm]]
+    ; CHECK: RET 0, $eax
+    %1:gr64 = COPY $rsi
+    %0:gr64 = COPY $rdi
+    %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope !5, !noalias !8)
+    %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope !5, !noalias !8)
+    MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope !10, !noalias !11)
+    MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope !10, !noalias !11)
+    %4:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    %5:gr32 = ADD32rm %4, %1, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    $eax = COPY %5
+    RET 0, $eax
+
+...
+---
+name:            test_mempcpy
+machineMetadataNodes:
+  - '!5 = !{!1, !6}'
+  - '!8 = !{!4, !9}'
+  - '!11 = !{!4, !6}'
+  - '!10 = !{!1, !9}'
+  - '!7 = distinct !{!7, !"MemcpyLoweringDomain"}'
+  - '!6 = distinct !{!6, !7, !"Src"}'
+  - '!9 = distinct !{!9, !7, !"Dst"}'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $rdi, $rsi
+
+    ; CHECK-LABEL: name: test_mempcpy
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+    ; CHECK: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm [[COPY1]], 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 1, !alias.scope !5, !noalias !8)
+    ; CHECK: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm [[COPY1]], 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 1, !alias.scope !5, !noalias !8)
+    ; CHECK: MOV64mr [[COPY1]], 1, $noreg, 8, $noreg, killed [[MOV64rm1]] :: (store 8 into %ir.p0 + 8, align 1, !alias.scope !10, !noalias !11)
+    ; CHECK: MOV64mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV64rm]] :: (store 8 into %ir.p0, align 1, !alias.scope !10, !noalias !11)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    ; CHECK: [[ADD32rm:%[0-9]+]]:gr32 = ADD32rm [[MOV32rm]], [[COPY]], 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    ; CHECK: $eax = COPY [[ADD32rm]]
+    ; CHECK: RET 0, $eax
+    %1:gr64 = COPY $rsi
+    %0:gr64 = COPY $rdi
+    %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 1, !alias.scope !5, !noalias !8)
+    %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 1, !alias.scope !5, !noalias !8)
+    MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 1, !alias.scope !10, !noalias !11)
+    MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 1, !alias.scope !10, !noalias !11)
+    %4:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from %ir.q, !alias.scope !3, !noalias !0)
+    %5:gr32 = ADD32rm %4, %1, 1, $noreg, 4, $noreg, implicit-def dead $eflags :: (load 4 from %ir.q1, !alias.scope !3, !noalias !0)
+    $eax = COPY %5
+    RET 0, $eax
+
+...