[AArch64] Legalize MVT::i64x8 in DAG isel lowering.

[labrinea]

Summary

This patch legalizes the Machine Value Type introduced in https://reviews.llvm.org/D94096 for loads and stores. It's notable to point out that the function that chooses an EVT for a given llvm type now needs to be specialized (dynamically binded) per target, as for AArch64 and only when the LS64 extension is present I am mapping [8 x i64] to MVT::i64x8, which would otherwise be MVT::Other since it's an aggregate type.

[t.p.northover]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

8251

This looks like it turns all [8 x i64] IR types into i64x8, which is pretty risky.

At the very least it means we probably need to add new tests for any natural IR operation you can do to that type (load, store, extract, insert, alloca, GEP, phi, select, function calls & returns and anything else you can think of to throw at it).

[efriedma]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

8251

To be clear, the risky part is making getValueType() do this universally, for every place where you can write [8 x i64] in LLVM IR. Adding a target hook specifically for inline asm operands should be fine.

[labrinea]

Changes since last revision:

• getAsmOperandValueType() is a target hook called only when the selection DAG is processing Inline Asm nodes
• GlobalISel was crashing before and now it just falls back to Selection DAG.

[labrinea]

ping

[efriedma]

llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp

347

It's a little unfortunate we're sticking this in target-independent code, but not sure there's a better place.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

251

Not sure why you're marking BITCAST Legal; there isn't any legal type to bitcast from.

254

I guess LOAD/STORE come out of type legalization?

[labrinea]

llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp

347

Not, I am afraid.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

251

I think that's a leftover from previous efforts, so as the UNDEF just below. I'll remove them.

254

These are the custom lowering operations for the load/store i512 instruction that clang inserts pre/post inline assembly for accessing the asm operands (see lines 4569 and 4595).

8254

This should be TargetLowering::getAsmOperandValueType. I'll fix it.

[labrinea]

Changes in this revision:

• Removed BITCAST and UNDEF from the legal operations on MVT::i64x8
• Rectified the derived target hook getAsmOperandValueType() to call the parent as a fallback instead of getValueType()

[efriedma]

LGTM

llvm/include/llvm/CodeGen/TargetLowering.h

do {
       NVT = (MVT::SimpleValueType)(NVT.SimpleTy+1);
       assert(NVT.isInteger() == VT.isInteger() && NVT != MVT::isVoid &&
              "Didn't find type to promote to!");
     } while (!isTypeLegal(NVT) ||
               getOperationAction(Op, NVT) == Promote);
     return NVT;
   }
 
+  virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,
+                                     bool AllowUnknown = false) const {
+    return getValueType(DL, Ty, AllowUnknown);
+  }
+
   /// Return the EVT corresponding to this LLVM type.  This is fixed by the LLVM
   /// operations except for the pointer size.  If AllowUnknown is true, this
   /// will return MVT::Other for types with no EVT counterpart (e.g. structs),
   /// otherwise it will assert.
   EVT getValueType(const DataLayout &DL, Type *Ty,
                    bool AllowUnknown = false) const {
     // Lower scalar pointers to native pointer types.
     if (auto *PTy = dyn_cast<PointerType>(Ty))

llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp

if (OpInfo.Type == InlineAsm::isInput ||
 
       // FIXME: Support aggregate input operands
       if (!OpTy->isSingleValueType()) {
         LLVM_DEBUG(
             dbgs() << "Aggregate input operands are not supported yet\n");
         return false;
       }
 
-      OpInfo.ConstraintVT = TLI->getValueType(DL, OpTy, true).getSimpleVT();
+      OpInfo.ConstraintVT =
+          TLI->getAsmOperandValueType(DL, OpTy, true).getSimpleVT();
 
     } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
       assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
       if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
         OpInfo.ConstraintVT =
             TLI->getSimpleValueType(DL, STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
-        OpInfo.ConstraintVT = TLI->getSimpleValueType(DL, Call.getType());
+        OpInfo.ConstraintVT =
+            TLI->getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
       }
       ++ResNo;
     } else {
       OpInfo.ConstraintVT = MVT::Other;
     }
 
+    if (OpInfo.ConstraintVT == MVT::i64x8)
+      return false;
+
     // Compute the constraint code and ConstraintType to use.
     computeConstraintToUse(TLI, OpInfo);
 
     // The selected constraint type might expose new sideeffects
     ExtraInfo.update(OpInfo);
   }
 
   // At this point, all operand types are decided.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

if (!OpTy->isSingleValueType() && OpTy->isSized()) {
       case 32:
       case 64:
       case 128:
         OpTy = IntegerType::get(Context, BitSize);
         break;
       }
     }
 
-    return TLI.getValueType(DL, OpTy, true);
+    return TLI.getAsmOperandValueType(DL, OpTy, true);
   }
 };
 
 
 } // end anonymous namespace
 
 /// Make sure that the output operand \p OpInfo and its corresponding input
 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
if (OpInfo.Type == InlineAsm::isInput ||
       // The return value of the call is this value.  As such, there is no
       // corresponding argument.
       assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
       if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
         OpInfo.ConstraintVT = TLI.getSimpleValueType(
             DAG.getDataLayout(), STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
-        OpInfo.ConstraintVT =
-            TLI.getSimpleValueType(DAG.getDataLayout(), Call.getType());
+        OpInfo.ConstraintVT = TLI.getAsmOperandValueType(
+            DAG.getDataLayout(), Call.getType()).getSimpleVT();
       }
       ++ResNo;
     } else {
       OpInfo.ConstraintVT = MVT::Other;
     }
 
     if (OpInfo.hasMatchingInput())
       ++NumMatchingOps;

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

case InlineAsm::isOutput:
       // The return value of the call is this value.  As such, there is no
       // corresponding argument.
       assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
       if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
         OpInfo.ConstraintVT =
             getSimpleValueType(DL, STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
-        OpInfo.ConstraintVT = getSimpleValueType(DL, Call.getType());
+        OpInfo.ConstraintVT =
+            getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
       }
       ++ResNo;
       break;
     case InlineAsm::isInput:
       OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
       break;
     case InlineAsm::isClobber:
       // Nothing to do.

llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
   default:
     return true; // Unknown mode.
   case 'w':
     Reg = getWRegFromXReg(Reg);
     break;
   case 'x':
     Reg = getXRegFromWReg(Reg);
     break;
+  case 't':
+    Reg = getXRegFromXRegTuple(Reg);
+    break;
   }
 
   O << AArch64InstPrinter::getRegisterName(Reg);
   return false;
 }
 
 // Prints the register in MO using class RC using the offset in the
 // new register class. This should not be used for cross class
bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
   if (MO.isReg()) {
     Register Reg = MO.getReg();
 
     // If this is a w or x register, print an x register.
     if (AArch64::GPR32allRegClass.contains(Reg) ||
         AArch64::GPR64allRegClass.contains(Reg))
       return printAsmMRegister(MO, 'x', O);
 
+    // If this is an x register tuple, print an x register.
+    if (AArch64::GPR64x8ClassRegClass.contains(Reg))
+      return printAsmMRegister(MO, 't', O);
+
     unsigned AltName = AArch64::NoRegAltName;
     const TargetRegisterClass *RegClass;
     if (AArch64::ZPRRegClass.contains(Reg)) {
       RegClass = &AArch64::ZPRRegClass;
     } else if (AArch64::PPRRegClass.contains(Reg)) {
       RegClass = &AArch64::PPRRegClass;
     } else {
       RegClass = &AArch64::FPR128RegClass;

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   CTLZ_MERGE_PASSTHRU,
   CTPOP_MERGE_PASSTHRU,
   DUP_MERGE_PASSTHRU,
   INDEX_VECTOR,
 
   // Cast between vectors of the same element type but differ in length.
   REINTERPRET_CAST,
 
+  // Nodes to build an LD64B / ST64B 64-bit quantity out of i64, and vice versa
+  LS64_BUILD,
+  LS64_EXTRACT,
+
   LD1_MERGE_ZERO,
   LD1S_MERGE_ZERO,
   LDNF1_MERGE_ZERO,
   LDNF1S_MERGE_ZERO,
   LDFF1_MERGE_ZERO,
   LDFF1S_MERGE_ZERO,
   LD1RQ_MERGE_ZERO,
   LD1RO_MERGE_ZERO,
public:
   unsigned getRedZoneSize(const Function &F) const {
     if (F.hasFnAttribute(Attribute::NoRedZone))
       return 0;
     return 128;
   }
 
   bool isAllActivePredicate(SDValue N) const;
 
+  EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,
+                             bool AllowUnknown = false) const override;
+
 private:
   /// Keep a pointer to the AArch64Subtarget around so that we can
   /// make the right decision when generating code for different targets.
   const AArch64Subtarget *Subtarget;
 
   bool isExtFreeImpl(const Instruction *Ext) const override;
 
   void addTypeForNEON(MVT VT, MVT PromotedBitwiseVT);

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   // When comparing vectors the result sets the different elements in the
   // vector to all-one or all-zero.
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
   // Set up the register classes.
   addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
   addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
 
+  if (Subtarget->hasLS64()) {
+    addRegisterClass(MVT::i64x8, &AArch64::GPR64x8ClassRegClass);
+    setOperationAction(ISD::BITCAST, MVT::i64x8, Legal);
+    setOperationAction(ISD::UNDEF, MVT::i64x8, Legal);
+    setOperationAction(ISD::LOAD, MVT::i64x8, Custom);
+    setOperationAction(ISD::STORE, MVT::i64x8, Custom);
+  }
+
   if (Subtarget->hasFPARMv8()) {
     addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
     addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
     addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
     addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
     addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
   }
 
case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::ORV_PRED)
     MAKE_CASE(AArch64ISD::EORV_PRED)
     MAKE_CASE(AArch64ISD::ANDV_PRED)
     MAKE_CASE(AArch64ISD::CLASTA_N)
     MAKE_CASE(AArch64ISD::CLASTB_N)
     MAKE_CASE(AArch64ISD::LASTA)
     MAKE_CASE(AArch64ISD::LASTB)
     MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
+    MAKE_CASE(AArch64ISD::LS64_BUILD)
+    MAKE_CASE(AArch64ISD::LS64_EXTRACT)
     MAKE_CASE(AArch64ISD::TBL)
     MAKE_CASE(AArch64ISD::FADD_PRED)
     MAKE_CASE(AArch64ISD::FADDA_PRED)
     MAKE_CASE(AArch64ISD::FADDV_PRED)
     MAKE_CASE(AArch64ISD::FDIV_PRED)
     MAKE_CASE(AArch64ISD::FMA_PRED)
     MAKE_CASE(AArch64ISD::FMAX_PRED)
     MAKE_CASE(AArch64ISD::FMAXV_PRED)
if (VT.isVector()) {
     SDValue Hi =
         DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i64, StoreNode->getValue(),
                     DAG.getConstant(1, Dl, MVT::i64));
     SDValue Result = DAG.getMemIntrinsicNode(
         AArch64ISD::STP, Dl, DAG.getVTList(MVT::Other),
         {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
         StoreNode->getMemoryVT(), StoreNode->getMemOperand());
     return Result;
+  } else if (MemVT == MVT::i64x8) {
+    SDValue Value = StoreNode->getValue();
+    assert(Value->getValueType(0) == MVT::i64x8);
+    SDValue Chain = StoreNode->getChain();
+    SDValue Base = StoreNode->getBasePtr();
+    EVT PtrVT = Base.getValueType();
+    for (unsigned i = 0; i < 8; i++) {
+      SDValue Part = DAG.getNode(AArch64ISD::LS64_EXTRACT, Dl, MVT::i64,
+                                 Value, DAG.getConstant(i, Dl, MVT::i32));
+      SDValue Ptr = DAG.getNode(ISD::ADD, Dl, PtrVT, Base,
+                                DAG.getConstant(i * 8, Dl, PtrVT));
+      Chain = DAG.getStore(Chain, Dl, Part, Ptr, StoreNode->getPointerInfo(),
+                           StoreNode->getOriginalAlign());
+    }
+    return Chain;
   }
 
   return SDValue();
 }
 
-// Custom lowering for extending v4i8 vector loads.
 SDValue AArch64TargetLowering::LowerLOAD(SDValue Op,
                                          SelectionDAG &DAG) const {
   SDLoc DL(Op);
   LoadSDNode *LoadNode = cast<LoadSDNode>(Op);
   assert(LoadNode && "Expected custom lowering of a load node");
+
+  if (LoadNode->getMemoryVT() == MVT::i64x8) {
+    SmallVector<SDValue, 8> Ops;
+    SDValue Base = LoadNode->getBasePtr();
+    SDValue Chain = LoadNode->getChain();
+    EVT PtrVT = Base.getValueType();
+    for (unsigned i = 0; i < 8; i++) {
+      SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
+                                DAG.getConstant(i * 8, DL, PtrVT));
+      SDValue Part = DAG.getLoad(MVT::i64, DL, Chain, Ptr,
+                                 LoadNode->getPointerInfo(),
+                                 LoadNode->getOriginalAlign());
+      Ops.push_back(Part);
+      Chain = SDValue(Part.getNode(), 1);
+    }
+    SDValue Loaded = DAG.getNode(AArch64ISD::LS64_BUILD, DL, MVT::i64x8, Ops);
+    return DAG.getMergeValues({Loaded, Chain}, DL);
+  }
+
+  // Custom lowering for extending v4i8 vector loads.
   EVT VT = Op->getValueType(0);
   assert((VT == MVT::v4i16 || VT == MVT::v4i32) && "Expected v4i16 or v4i32");
 
   if (LoadNode->getMemoryVT() != MVT::v4i8)
     return SDValue();
 
   unsigned ExtType;
   if (LoadNode->getExtensionType() == ISD::SEXTLOAD)

 std::pair<unsigned, const TargetRegisterClass *>
 AArch64TargetLowering::getRegForInlineAsmConstraint(
     const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'r':
       if (VT.isScalableVector())
         return std::make_pair(0U, nullptr);
+      if (Subtarget->hasLS64() && VT.getSizeInBits() == 512)
+        return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
       if (VT.getFixedSizeInBits() == 64)
         return std::make_pair(0U, &AArch64::GPR64commonRegClass);
       return std::make_pair(0U, &AArch64::GPR32commonRegClass);
     case 'w': {
       if (!Subtarget->hasFPARMv8())
         break;
       if (VT.isScalableVector()) {
         if (VT.getVectorElementType() != MVT::i1)
AArch64TargetLowering::getRegForInlineAsmConstraint(
   if (Res.second && !Subtarget->hasFPARMv8() &&
       !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
       !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
     return std::make_pair(0U, nullptr);
 
   return Res;
 }
 
+EVT AArch64TargetLowering::getAsmOperandValueType(const DataLayout &DL,
+                                                  llvm::Type *Ty,
+                                                  bool AllowUnknown) const {
+  if (Subtarget->hasLS64() && Ty->isIntegerTy(512))
+    return EVT(MVT::i64x8);
+
+  return TargetLowering::getValueType(DL, Ty, AllowUnknown);
+}
+
 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
 /// vector.  If it is invalid, don't add anything to Ops.
 void AArch64TargetLowering::LowerAsmOperandForConstraint(
     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
     SelectionDAG &DAG) const {
   SDValue Result;
 
   // Currently only support length 1 constraints.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

let AddedComplexity = 10 in {
   def : dot_v8i8<UDOTv8i8, AArch64umull, zext>;
   def : dot_v16i8<SDOTv16i8, AArch64smull, sext>;
   def : dot_v16i8<UDOTv16i8, AArch64umull, zext>;
 
   // FIXME: add patterns to generate vector by element dot product.
   // FIXME: add SVE dot-product patterns.
 }
 
+// Custom DAG nodes and isel rules to make a 64-byte block out of eight GPRs,
+// so that it can be used as input to inline asm, and vice versa.
+def LS64_BUILD : SDNode<"AArch64ISD::LS64_BUILD", SDTypeProfile<1, 8, []>>;
+def LS64_EXTRACT : SDNode<"AArch64ISD::LS64_EXTRACT", SDTypeProfile<1, 2, []>>;
+def : Pat<(i64x8 (LS64_BUILD GPR64:$x0, GPR64:$x1, GPR64:$x2, GPR64:$x3,
+                             GPR64:$x4, GPR64:$x5, GPR64:$x6, GPR64:$x7)),
+          (REG_SEQUENCE GPR64x8Class,
+              $x0, x8sub_0, $x1, x8sub_1, $x2, x8sub_2, $x3, x8sub_3,
+              $x4, x8sub_4, $x5, x8sub_5, $x6, x8sub_6, $x7, x8sub_7)>;
+foreach i = 0-7 in {
+  def : Pat<(i64 (LS64_EXTRACT (i64x8 GPR64x8:$val), (i32 i))),
+            (EXTRACT_SUBREG $val, !cast<SubRegIndex>("x8sub_"#i))>;
+}
+
 let Predicates = [HasLS64] in {
   def LD64B: LoadStore64B<0b101, "ld64b", (ins GPR64sp:$Rn),
                                           (outs GPR64x8:$Rt)>;
   def ST64B: LoadStore64B<0b001, "st64b", (ins GPR64x8:$Rt, GPR64sp:$Rn),
                                           (outs)>;
   def ST64BV:   Store64BV<0b011, "st64bv">;
   def ST64BV0:  Store64BV<0b010, "st64bv0">;
 

llvm/lib/Target/AArch64/AArch64RegisterInfo.td

let Namespace = "AArch64" in {
   foreach i = 0-7 in
     def "x8sub_"#i : SubRegIndex<64, !mul(64, i)>;
 }
 
 def Tuples8X : RegisterTuples<
   !foreach(i, [0,1,2,3,4,5,6,7], !cast<SubRegIndex>("x8sub_"#i)),
   !foreach(i, [0,1,2,3,4,5,6,7], (trunc (decimate (rotl GPR64, i), 2), 12))>;
 
-def GPR64x8Class : RegisterClass<"AArch64", [i64], 64, (trunc Tuples8X, 12)>;
+def GPR64x8Class : RegisterClass<"AArch64", [i64x8], 512, (trunc Tuples8X, 12)> {
+  let Size = 512;
+}
 def GPR64x8AsmOp : AsmOperandClass {
   let Name = "GPR64x8";
   let ParserMethod = "tryParseGPR64x8";
   let RenderMethod = "addRegOperands";
 }
 def GPR64x8 : RegisterOperand<GPR64x8Class, "printGPR64x8"> {
   let ParserMatchClass = GPR64x8AsmOp;
   let PrintMethod = "printGPR64x8";

llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h

inline static unsigned getXRegFromWReg(unsigned Reg) {
   case AArch64::W30: return AArch64::LR;
   case AArch64::WSP: return AArch64::SP;
   case AArch64::WZR: return AArch64::XZR;
   }
   // For anything else, return it unchanged.
   return Reg;
 }
 
+inline static unsigned getXRegFromXRegTuple(unsigned RegTuple) {
+  switch (RegTuple) {
+  case AArch64::X0_X1_X2_X3_X4_X5_X6_X7: return AArch64::X0;
+  case AArch64::X2_X3_X4_X5_X6_X7_X8_X9: return AArch64::X2;
+  case AArch64::X4_X5_X6_X7_X8_X9_X10_X11: return AArch64::X4;
+  case AArch64::X6_X7_X8_X9_X10_X11_X12_X13: return AArch64::X6;
+  case AArch64::X8_X9_X10_X11_X12_X13_X14_X15: return AArch64::X8;
+  case AArch64::X10_X11_X12_X13_X14_X15_X16_X17: return AArch64::X10;
+  case AArch64::X12_X13_X14_X15_X16_X17_X18_X19: return AArch64::X12;
+  case AArch64::X14_X15_X16_X17_X18_X19_X20_X21: return AArch64::X14;
+  case AArch64::X16_X17_X18_X19_X20_X21_X22_X23: return AArch64::X16;
+  case AArch64::X18_X19_X20_X21_X22_X23_X24_X25: return AArch64::X18;
+  case AArch64::X20_X21_X22_X23_X24_X25_X26_X27: return AArch64::X20;
+  case AArch64::X22_X23_X24_X25_X26_X27_X28_FP: return AArch64::X22;
+  }
+  // For anything else, return it unchanged.
+  return RegTuple;
+}
+
 static inline unsigned getBRegFromDReg(unsigned Reg) {
   switch (Reg) {
   case AArch64::D0:  return AArch64::B0;
   case AArch64::D1:  return AArch64::B1;
   case AArch64::D2:  return AArch64::B2;
   case AArch64::D3:  return AArch64::B3;
   case AArch64::D4:  return AArch64::B4;
   case AArch64::D5:  return AArch64::B5;

llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll

 define void @asm_indirect_output() {
 entry:
   %ap = alloca i8*, align 8
   %0 = load i8*, i8** %ap, align 8
   call void asm sideeffect "", "=*r|m,0,~{memory}"(i8** %ap, i8* %0)
   ret void
 }
 
+%struct.foo = type { [8 x i64] }
+
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction:{{.*}}ld64b{{.*}}asm_output_ls64
+; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for asm_output_ls64
+; FALLBACK-WITH-REPORT-OUT-LABEL: asm_output_ls64
+define void @asm_output_ls64(%struct.foo* %output, i8* %addr) #2 {
+entry:
+  %val = call i512 asm sideeffect "ld64b $0,[$1]", "=r,r,~{memory}"(i8* %addr)
+  %outcast = bitcast %struct.foo* %output to i512*
+  store i512 %val, i512* %outcast, align 8
+  ret void
+}
+
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction:{{.*}}st64b{{.*}}asm_input_ls64
+; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for asm_input_ls64
+; FALLBACK-WITH-REPORT-OUT-LABEL: asm_input_ls64
+define void @asm_input_ls64(%struct.foo* %input, i8* %addr) #2 {
+entry:
+  %incast = bitcast %struct.foo* %input to i512*
+  %val = load i512, i512* %incast, align 8
+  call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 %val, i8* %addr)
+  ret void
+}
+
 attributes #1 = { "target-features"="+sve" }
+attributes #2 = { "target-features"="+ls64" }
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 %pattern)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*)

llvm/test/CodeGen/AArch64/ls64-inline-asm.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+ls64 -verify-machineinstrs -o - %s | FileCheck %s
+
+%struct.foo = type { [8 x i64] }
+
+define void @load(%struct.foo* %output, i8* %addr) {
+; CHECK-LABEL: load:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    ld64b x2, [x1]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    stp x8, x9, [x0, #48]
+; CHECK-NEXT:    stp x6, x7, [x0, #32]
+; CHECK-NEXT:    stp x4, x5, [x0, #16]
+; CHECK-NEXT:    stp x2, x3, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %val = call i512 asm sideeffect "ld64b $0,[$1]", "=r,r,~{memory}"(i8* %addr)
+  %outcast = bitcast %struct.foo* %output to i512*
+  store i512 %val, i512* %outcast, align 8
+  ret void
+}
+
+define void @store(%struct.foo* %input, i8* %addr) {
+; CHECK-LABEL: store:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp x8, x9, [x0, #48]
+; CHECK-NEXT:    ldp x6, x7, [x0, #32]
+; CHECK-NEXT:    ldp x4, x5, [x0, #16]
+; CHECK-NEXT:    ldp x2, x3, [x0]
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    st64b x2, [x1]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    ret
+entry:
+  %incast = bitcast %struct.foo* %input to i512*
+  %val = load i512, i512* %incast, align 8
+  call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 %val, i8* %addr)
+  ret void
+}
+
+define void @store2(i32* %in, i8* %addr) {
+; CHECK-LABEL: store2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #64 // =64
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    ldpsw x2, x3, [x0]
+; CHECK-NEXT:    ldrsw x4, [x0, #16]
+; CHECK-NEXT:    ldrsw x5, [x0, #64]
+; CHECK-NEXT:    ldrsw x6, [x0, #100]
+; CHECK-NEXT:    ldrsw x7, [x0, #144]
+; CHECK-NEXT:    ldrsw x8, [x0, #196]
+; CHECK-NEXT:    ldrsw x9, [x0, #256]
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    st64b x2, [x1]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    add sp, sp, #64 // =64
+; CHECK-NEXT:    ret
+entry:
+  %0 = load i32, i32* %in, align 4
+  %conv = sext i32 %0 to i64
+  %arrayidx1 = getelementptr inbounds i32, i32* %in, i64 1
+  %1 = load i32, i32* %arrayidx1, align 4
+  %conv2 = sext i32 %1 to i64
+  %arrayidx4 = getelementptr inbounds i32, i32* %in, i64 4
+  %2 = load i32, i32* %arrayidx4, align 4
+  %conv5 = sext i32 %2 to i64
+  %arrayidx7 = getelementptr inbounds i32, i32* %in, i64 16
+  %3 = load i32, i32* %arrayidx7, align 4
+  %conv8 = sext i32 %3 to i64
+  %arrayidx10 = getelementptr inbounds i32, i32* %in, i64 25
+  %4 = load i32, i32* %arrayidx10, align 4
+  %conv11 = sext i32 %4 to i64
+  %arrayidx13 = getelementptr inbounds i32, i32* %in, i64 36
+  %5 = load i32, i32* %arrayidx13, align 4
+  %conv14 = sext i32 %5 to i64
+  %arrayidx16 = getelementptr inbounds i32, i32* %in, i64 49
+  %6 = load i32, i32* %arrayidx16, align 4
+  %conv17 = sext i32 %6 to i64
+  %arrayidx19 = getelementptr inbounds i32, i32* %in, i64 64
+  %7 = load i32, i32* %arrayidx19, align 4
+  %conv20 = sext i32 %7 to i64
+  %s.sroa.10.0.insert.ext = zext i64 %conv20 to i512
+  %s.sroa.10.0.insert.shift = shl nuw i512 %s.sroa.10.0.insert.ext, 448
+  %s.sroa.9.0.insert.ext = zext i64 %conv17 to i512
+  %s.sroa.9.0.insert.shift = shl nuw nsw i512 %s.sroa.9.0.insert.ext, 384
+  %s.sroa.9.0.insert.insert = or i512 %s.sroa.10.0.insert.shift, %s.sroa.9.0.insert.shift
+  %s.sroa.8.0.insert.ext = zext i64 %conv14 to i512
+  %s.sroa.8.0.insert.shift = shl nuw nsw i512 %s.sroa.8.0.insert.ext, 320
+  %s.sroa.8.0.insert.insert = or i512 %s.sroa.9.0.insert.insert, %s.sroa.8.0.insert.shift
+  %s.sroa.7.0.insert.ext = zext i64 %conv11 to i512
+  %s.sroa.7.0.insert.shift = shl nuw nsw i512 %s.sroa.7.0.insert.ext, 256
+  %s.sroa.7.0.insert.insert = or i512 %s.sroa.8.0.insert.insert, %s.sroa.7.0.insert.shift
+  %s.sroa.6.0.insert.ext = zext i64 %conv8 to i512
+  %s.sroa.6.0.insert.shift = shl nuw nsw i512 %s.sroa.6.0.insert.ext, 192
+  %s.sroa.6.0.insert.insert = or i512 %s.sroa.7.0.insert.insert, %s.sroa.6.0.insert.shift
+  %s.sroa.5.0.insert.ext = zext i64 %conv5 to i512
+  %s.sroa.5.0.insert.shift = shl nuw nsw i512 %s.sroa.5.0.insert.ext, 128
+  %s.sroa.4.0.insert.ext = zext i64 %conv2 to i512
+  %s.sroa.4.0.insert.shift = shl nuw nsw i512 %s.sroa.4.0.insert.ext, 64
+  %s.sroa.4.0.insert.mask = or i512 %s.sroa.6.0.insert.insert, %s.sroa.5.0.insert.shift
+  %s.sroa.0.0.insert.ext = zext i64 %conv to i512
+  %s.sroa.0.0.insert.mask = or i512 %s.sroa.4.0.insert.mask, %s.sroa.4.0.insert.shift
+  %s.sroa.0.0.insert.insert = or i512 %s.sroa.0.0.insert.mask, %s.sroa.0.0.insert.ext
+  call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 %s.sroa.0.0.insert.insert, i8* %addr)
+  ret void
+}
+