[AArch64] Add support for MTE intrinsics

[javed.absar]

Summary

This patch provides intrinsics support for Memory Tagging Extension (MTE),
which was introduced with the Armv8.5-a architecture.
This is llvm patch. The corresponding clang patch is: https://reviews.llvm.org/D60485

These intrinsics are available when __ARM_FEATURE_MEMORY_TAGGING is defined.
Each intrinsic is described in detail in the latest ACLE Q1 2019 documentation:

https://developer.arm.com/docs/101028/latest

However, below we also list the intrinsics:

1. T* __arm_mte_create_random_tag(T* src, uint64_t mask); This intrinsic returns a pointer containing a randomly created logical address tag.
2. T* __arm_mte_increment_tag(T* src, unsigned offset); This intrinsic returns a pointer which is a copy of the input pointer src but with the logical address tag part offset by a specified offset value.
3. uint64_t __arm_mte_exclude_tag(T* src, uint64_t excluded); This intrinsic adds a logical tag to the set of excluded logical tags.
4. void __arm_mte_set_tag(T* tag_address); This intrinsic stores an allocation tag, computed from the logical tag, to the tag memory thereby setting the allocation tag for the 16-byte granule of memory.
5. T* __arm_mte_get_tag(T* address); This intrinsic loads the allocation tag from tag memory and returns the corresponding logical tag as part of the returned pointer value.
6. ptrdiff_t __arm_mte_ptrdiff(T* a, T* b); The intrinsic calculates the difference between the address parts of the two pointers, ignoring the tags.

[DavidSpickett]

One nit otherwise LGTM.

include/llvm/IR/IntrinsicsAArch64.td

690

Remove this.

[javed.absar]

Fixed as suggested

[eugenis]

llvm/trunk/include/llvm/IR/IntrinsicsAArch64.td

692 ↗

(On Diff #196201)

Sorry for such a late question.
Why is IRG not IntrNoMem? Does this model the fact that it updates the internal state of the RNG in the CPU?

It is not covered by tests.

[chill]

llvm/trunk/include/llvm/IR/IntrinsicsAArch64.td

692 ↗

(On Diff #196201)

IntrNoMem would be incorrect, as that means also no side effects whatsoever, and IRG potentially has side effects.
On the other hand it, in fact does not modify memory, so IntrInaccessibleMemOnly is pessimistic and I can't see how being pessimistic in this case ensures correctness (the only good reason for being pessimistic IMHO).
The spot on attribute would be IntrHasSideEffects.

[chill]

llvm/trunk/test/CodeGen/AArch64/arm64-mte.ll

12 ↗

(On Diff #196201)

IRG is mentioned here.

[eugenis]

llvm/trunk/include/llvm/IR/IntrinsicsAArch64.td

692 ↗

(On Diff #196201)

Sounds good, I can prepare a change.

llvm/trunk/test/CodeGen/AArch64/arm64-mte.ll

12 ↗

(On Diff #196201)

What I meant is, there are no tests that notice the change in the intrinsic attribute.

include/llvm/IR/IntrinsicsAArch64.td

def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
     [IntrNoMem]>;
 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
     [IntrNoMem]>;
 def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
     [IntrNoMem]>;
 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
     [IntrNoMem]>;
 }
+
+//===----------------------------------------------------------------------===//
+// Memory Tagging Extensions (MTE) Intrinsics
+// SDCOMP-51339
+let TargetPrefix = "aarch64" in {
+def int_aarch64_irg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
+    [IntrInaccessibleMemOnly]>;
+def int_aarch64_addg  : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
+    [IntrNoMem]>;
+def int_aarch64_gmi   : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
+    [IntrNoMem]>;
+def int_aarch64_ldg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
+    [IntrReadMem]>;
+def int_aarch64_stg   : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
+    [IntrWriteMem]>;
+def int_aarch64_subp :  Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
+    [IntrNoMem]>;
+}

lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
     return SelectAddrModeIndexed7S(N, 4, Base, OffImm);
   }
   bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
     return SelectAddrModeIndexed7S(N, 8, Base, OffImm);
   }
   bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
     return SelectAddrModeIndexed7S(N, 16, Base, OffImm);
   }
+  bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
+    return SelectAddrModeIndexedBitWidth(N, true, 9, 16, Base, OffImm);
+  }
+  bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
+    return SelectAddrModeIndexedBitWidth(N, false, 6, 16, Base, OffImm);
+  }
   bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
     return SelectAddrModeIndexed(N, 1, Base, OffImm);
   }
   bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
     return SelectAddrModeIndexed(N, 2, Base, OffImm);
   }
   bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
     return SelectAddrModeIndexed(N, 4, Base, OffImm);

 
 // Include the pieces autogenerated from the target description.
 #include "AArch64GenDAGISel.inc"
 
 private:
   bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
                              SDValue &Shift);
   bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
+                               SDValue &OffImm) {
+    return SelectAddrModeIndexedBitWidth(N, true, 7, Size, Base, OffImm);
+  }
+  bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
+                                     unsigned Size, SDValue &Base,
                                      SDValue &OffImm);
   bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
                              SDValue &OffImm);
   bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
                               SDValue &OffImm);
   bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
                          SDValue &Offset, SDValue &SignExtend,
                          SDValue &DoShift);
   bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
for (auto Use : N->uses()) {
     // register).
     if (isStrongerThanMonotonic(cast<MemSDNode>(Use)->getOrdering()))
       return false;
   }
 
   return true;
 }
 
-/// SelectAddrModeIndexed7S - Select a "register plus scaled signed 7-bit
+/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
 /// immediate" address.  The "Size" argument is the size in bytes of the memory
 /// reference, which determines the scale.
-bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(SDValue N, unsigned Size,
+bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
+                                                        unsigned BW, unsigned Size,
                                                         SDValue &Base,
                                                         SDValue &OffImm) {
   SDLoc dl(N);
   const DataLayout &DL = CurDAG->getDataLayout();
   const TargetLowering *TLI = getTargetLowering();
   if (N.getOpcode() == ISD::FrameIndex) {
     int FI = cast<FrameIndexSDNode>(N)->getIndex();
     Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
     OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
     return true;
   }
 
   // As opposed to the (12-bit) Indexed addressing mode below, the 7-bit signed
   // selected here doesn't support labels/immediates, only base+offset.
-
   if (CurDAG->isBaseWithConstantOffset(N)) {
     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
+      if (IsSignedImm) {
         int64_t RHSC = RHS->getSExtValue();
         unsigned Scale = Log2_32(Size);
-      if ((RHSC & (Size - 1)) == 0 && RHSC >= -(0x40 << Scale) &&
-          RHSC < (0x40 << Scale)) {
+        int64_t Range = 0x1 << (BW-1);
+
+        if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
+            RHSC < (Range << Scale)) {
           Base = N.getOperand(0);
           if (Base.getOpcode() == ISD::FrameIndex) {
             int FI = cast<FrameIndexSDNode>(Base)->getIndex();
             Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
           }
           OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
           return true;
         }
+      } else {
+        // unsigned Immediate
+        uint64_t RHSC = RHS->getZExtValue();
+        unsigned Scale = Log2_32(Size);
+        uint64_t Range = 0x1 << BW;
+
+        if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
+          Base = N.getOperand(0);
+          if (Base.getOpcode() == ISD::FrameIndex) {
+            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
+            Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
+          }
+          OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
+          return true;
+        }
+      }
     }
   }
-
   // Base only. The address will be materialized into a register before
   // the memory is accessed.
   //    add x0, Xbase, #offset
   //    stp x1, x2, [x0]
   Base = N;
   OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
   return true;
 }

lib/Target/AArch64/AArch64InstrFormats.td

 }
 
 def am_indexed7s8   : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S8", []>;
 def am_indexed7s16  : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S16", []>;
 def am_indexed7s32  : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S32", []>;
 def am_indexed7s64  : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S64", []>;
 def am_indexed7s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S128", []>;
 
+def am_indexedu6s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedU6S128", []>;
+def am_indexeds9s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedS9S128", []>;
+
 // uimm5sN predicate - True if the immediate is a multiple of N in the range
 // [0 * N, 32 * N].
 def UImm5s2Operand : UImmScaledMemoryIndexed<5, 2>;
 def UImm5s4Operand : UImmScaledMemoryIndexed<5, 4>;
 def UImm5s8Operand : UImmScaledMemoryIndexed<5, 8>;
 
 def uimm5s2 : Operand<i64>, ImmLeaf<i64,
                 [{ return Imm >= 0 && Imm < (32*2) && ((Imm % 2) == 0); }]> {
class AddSubG<bit isSub, string asm_inst, SDPatternOperator OpNode>
   let Inst{23-22} = 0b10;
   let Inst{21-16} = imm6;
   let Inst{15-14} = 0b00;
   let Inst{13-10} = imm4;
   let Unpredictable{15-14} = 0b11;
 }
 
 class SUBP<bit setsFlags, string asm_instr, SDPatternOperator OpNode>
-      : BaseTwoOperand<0b0000, GPR64, asm_instr, null_frag, GPR64sp, GPR64sp> {
+      : BaseTwoOperand<0b0000, GPR64, asm_instr, OpNode, GPR64sp, GPR64sp> {
   let Inst{31} = 1;
   let Inst{29} = setsFlags;
 }
 
 //---
 // Extract
 //---
 def SDTA64EXTR : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,

lib/Target/AArch64/AArch64InstrInfo.cpp

unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
   case AArch64::LDPWi:
   case AArch64::LDPSi:
   case AArch64::STPWi:
   case AArch64::STPSi:
   case AArch64::LDNPWi:
   case AArch64::LDNPSi:
   case AArch64::STNPWi:
   case AArch64::STNPSi:
+  case AArch64::LDG:
     return 3;
+  case AArch64::ADDG:
+  case AArch64::STGOffset:
+    return 2;
   }
 }
 
 bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     return false;
   // Scaled instructions.
bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
   case AArch64::LDRSBWui:
   case AArch64::LDRSBXui:
   case AArch64::STRBui:
   case AArch64::STRBBui:
     Scale = Width = 1;
     MinOffset = 0;
     MaxOffset = 4095;
     break;
+  case AArch64::ADDG:
+    Scale = 16;
+    Width = 0;
+    MinOffset = 0;
+    MaxOffset = 63;
+    break;
+  case AArch64::LDG:
+  case AArch64::STGOffset:
+    Scale = Width = 16;
+    MinOffset = -256;
+    MaxOffset = 255;
+    break;
   }
 
   return true;
 }
 
 static unsigned getOffsetStride(unsigned Opc) {
   switch (Opc) {
   default:

lib/Target/AArch64/AArch64InstrInfo.td

 defm : STOPregister<"stsmax","LDSMAX">;// STSMAXx
 defm : STOPregister<"stsmin","LDSMIN">;// STSMINx
 defm : STOPregister<"stumax","LDUMAX">;// STUMAXx
 defm : STOPregister<"stumin","LDUMIN">;// STUMINx
 
 // v8.5 Memory Tagging Extension
 let Predicates = [HasMTE] in {
 
-def IRG   : BaseTwoOperand<0b0100, GPR64sp, "irg", null_frag, GPR64sp, GPR64>,
+def IRG   : BaseTwoOperand<0b0100, GPR64sp, "irg", int_aarch64_irg, GPR64sp, GPR64>,
             Sched<[]>{
   let Inst{31} = 1;
 }
-def GMI   : BaseTwoOperand<0b0101, GPR64, "gmi", null_frag, GPR64sp>, Sched<[]>{
+def GMI   : BaseTwoOperand<0b0101, GPR64, "gmi", int_aarch64_gmi, GPR64sp>, Sched<[]>{
   let Inst{31} = 1;
   let isNotDuplicable = 1;
 }
 def ADDG  : AddSubG<0, "addg", null_frag>;
 def SUBG  : AddSubG<1, "subg", null_frag>;
 
 def : InstAlias<"irg $dst, $src", (IRG GPR64sp:$dst, GPR64sp:$src, XZR), 1>;
 
-def SUBP : SUBP<0, "subp", null_frag>, Sched<[]>;
+def SUBP : SUBP<0, "subp", int_aarch64_subp>, Sched<[]>;
 def SUBPS : SUBP<1, "subps", null_frag>, Sched<[]>{
   let Defs = [NZCV];
 }
 
 def : InstAlias<"cmpp $lhs, $rhs", (SUBPS XZR, GPR64sp:$lhs, GPR64sp:$rhs), 0>;
 
 def LDG : MemTagLoad<"ldg", "\t$Rt, [$Rn, $offset]">;
+
+def : Pat<(int_aarch64_addg (am_indexedu6s128 GPR64sp:$Rn, uimm6s16:$imm6), imm0_15:$imm4),
+          (ADDG GPR64sp:$Rn, imm0_63:$imm6, imm0_15:$imm4)>;
+def : Pat<(int_aarch64_ldg GPR64:$Rt, (am_indexeds9s128 GPR64sp:$Rn,  simm9s16:$offset)),
+          (LDG GPR64:$Rt, GPR64sp:$Rn,  simm9s16:$offset)>;
+
 def : InstAlias<"ldg $Rt, [$Rn]", (LDG GPR64:$Rt, GPR64sp:$Rn, 0), 1>;
 
 def LDGM : MemTagVector<1, "ldgm", "\t$Rt, [$Rn]",
                    (outs GPR64:$Rt), (ins GPR64sp:$Rn)>;
 def STGM : MemTagVector<0, "stgm", "\t$Rt, [$Rn]",
                    (outs), (ins GPR64:$Rt, GPR64sp:$Rn)>;
-
 def STZGM : MemTagVector<0, "stzgm", "\t$Rt, [$Rn]",
                    (outs), (ins GPR64:$Rt, GPR64sp:$Rn)> {
   let Inst{23} = 0;
 }
 
 defm STG   : MemTagStore<0b00, "stg">;
 defm STZG  : MemTagStore<0b01, "stzg">;
 defm ST2G  : MemTagStore<0b10, "st2g">;
 defm STZ2G : MemTagStore<0b11, "stz2g">;
 
 defm STGP     : StorePairOffset <0b01, 0, GPR64z, simm7s16, "stgp">;
 def  STGPpre  : StorePairPreIdx <0b01, 0, GPR64z, simm7s16, "stgp">;
 def  STGPpost : StorePairPostIdx<0b01, 0, GPR64z, simm7s16, "stgp">;
 
+def : Pat<(int_aarch64_stg GPR64:$Rt, (am_indexeds9s128 GPR64sp:$Rn, simm9s16:$offset)),
+          (STGOffset GPR64:$Rt, GPR64sp:$Rn,  simm9s16:$offset)>;
+
 } // Predicates = [HasMTE]
 
 //===----------------------------------------------------------------------===//
 // Logical instructions.
 //===----------------------------------------------------------------------===//
 
 // (immediate)
 defm ANDS : LogicalImmS<0b11, "ands", AArch64and_flag, "bics">;

test/CodeGen/AArch64/arm64-mte.ll

+; RUN: llc < %s -mtriple=arm64-eabi -mattr=+mte | FileCheck %s
+
+; test create_tag
+define i32* @create_tag(i32* %ptr, i32 %m) {
+entry:
+; CHECK-LABEL: create_tag:
+  %0 = bitcast i32* %ptr to i8*
+  %1 = zext i32 %m to i64
+  %2 = tail call i8* @llvm.aarch64.irg(i8* %0, i64 %1)
+  %3 = bitcast i8* %2 to i32*
+  ret i32* %3
+;CHECK: irg x0, x0, {{x[0-9]+}}
+}
+
+; *********** __arm_mte_increment_tag  *************
+; test increment_tag1
+define i32* @increment_tag1(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag1:
+  %0 = bitcast i32* %ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.addg(i8* %0, i64 7)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: addg x0, x0, #0, #7
+}
+
+%struct.S2K = type { [512 x i32] }
+define i32* @increment_tag1stack(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag1stack:
+  %s = alloca %struct.S2K, align 4
+  %0 = bitcast %struct.S2K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0)
+  %1 = call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7)
+  %2 = bitcast i8* %1 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0)
+  ret i32* %2
+; CHECK: addg x0, sp, #0, #7
+}
+
+
+define i32* @increment_tag2(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag2:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 4
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: addg x0, x0, #16, #7
+}
+
+define i32* @increment_tag2stack(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag2stack:
+  %s = alloca %struct.S2K, align 4
+  %0 = bitcast %struct.S2K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 4
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0)
+  ret i32* %3
+; CHECK: addg x0, sp, #16, #7
+}
+
+define i32* @increment_tag3(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag3:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 252
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: addg x0, x0, #1008, #7
+}
+
+define i32* @increment_tag3stack(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag3stack:
+  %s = alloca %struct.S2K, align 4
+  %0 = bitcast %struct.S2K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 252
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0)
+  ret i32* %3
+; CHECK: addg x0, sp, #1008, #7
+}
+
+
+define i32* @increment_tag4(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag4:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 256
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: add [[T0:x[0-9]+]], x0, #1024
+; CHECK-NEXT: addg x0, [[T0]], #0, #7
+}
+
+define i32* @increment_tag4stack(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag4stack:
+  %s = alloca %struct.S2K, align 4
+  %0 = bitcast %struct.S2K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 256
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0)
+  ret i32* %3
+; CHECK: add [[T0:x[0-9]+]], {{.*}}, #1024
+; CHECK-NEXT: addg x0, [[T0]], #0, #7
+}
+
+
+define i32* @increment_tag5(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag5:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 5
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: add [[T0:x[0-9]+]], x0, #20
+; CHECK-NEXT: addg x0, [[T0]], #0, #7
+}
+
+define i32* @increment_tag5stack(i32* %ptr) {
+entry:
+; CHECK-LABEL: increment_tag5stack:
+  %s = alloca %struct.S2K, align 4
+  %0 = bitcast %struct.S2K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 5
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0)
+  ret i32* %3
+; CHECK: add [[T0:x[0-9]+]], {{.*}}, #20
+; CHECK-NEXT: addg x0, [[T0]], #0, #7
+}
+
+
+; *********** __arm_mte_exclude_tag  *************
+; test exclude_tag
+define i32 @exclude_tag(i32* %ptr, i32 %m) local_unnamed_addr #0 {
+entry:
+;CHECK-LABEL: exclude_tag:
+  %0 = zext i32 %m to i64
+  %1 = bitcast i32* %ptr to i8*
+  %2 = tail call i64 @llvm.aarch64.gmi(i8* %1, i64 %0)
+  %conv = trunc i64 %2 to i32
+  ret i32 %conv
+; CHECK: gmi  x0, x0, {{x[0-9]+}}
+}
+
+
+; *********** __arm_mte_get_tag *************
+%struct.S8K = type { [2048 x i32] }
+define i32* @get_tag1(i32* %ptr) {
+entry:
+; CHECK-LABEL: get_tag1:
+  %0 = bitcast i32* %ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.ldg(i8* %0, i8* %0)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK ldg x0, [x0]
+}
+
+define i32* @get_tag1_two_parm(i32* %ret_ptr, i32* %ptr) {
+entry:
+; CHECK-LABEL: get_tag1_two_parm:
+  %0 = bitcast i32* %ret_ptr to i8*
+  %1 = bitcast i32* %ptr to i8*
+  %2 = tail call i8* @llvm.aarch64.ldg(i8* %0, i8* %1)
+  %3 = bitcast i8* %2 to i32*
+  ret i32* %3
+; CHECK ldg x0, [x1]
+}
+
+define i32* @get_tag1stack() {
+entry:
+; CHECK-LABEL: get_tag1stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %1 = call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0)
+  %2 = bitcast i8* %1 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret i32* %2
+; CHECK: mov [[T0:x[0-9]+]], sp
+; CHECK: ldg [[T0]], [sp]
+}
+
+define i32* @get_tag1stack_two_param(i32* %ret_ptr) {
+entry:
+; CHECK-LABEL: get_tag1stack_two_param:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  %1 = bitcast i32*  %ret_ptr to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %0)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret i32* %3
+; CHECK-NOT: mov {{.*}}, sp
+; CHECK: ldg x0, [sp]
+}
+
+
+define i32* @get_tag2(i32* %ptr) {
+entry:
+; CHECK-LABEL: get_tag2:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 4
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: add  [[T0:x[0-9]+]], x0, #16
+; CHECK: ldg  [[T0]], [x0, #16]
+}
+
+define i32* @get_tag2stack() {
+entry:
+; CHECK-LABEL: get_tag2stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 4
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret i32* %3
+; CHECK: mov [[T0:x[0-9]+]], sp
+; CHECK: add x0, [[T0]], #16
+; CHECK: ldg x0, [sp, #16]
+}
+
+
+define i32* @get_tag3(i32* %ptr) {
+entry:
+; CHECK-LABEL: get_tag3:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1020
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: add [[T0:x[0-8]+]], x0, #4080
+; CHECK: ldg [[T0]], [x0, #4080]
+}
+
+define i32* @get_tag3stack() {
+entry:
+; CHECK-LABEL: get_tag3stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1020
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret i32* %3
+; CHECK: mov [[T0:x[0-9]+]], sp
+; CHECK: add x0, [[T0]], #4080
+; CHECK: ldg x0, [sp, #4080]
+}
+
+
+define i32* @get_tag4(i32* %ptr) {
+entry:
+; CHECK-LABEL: get_tag4:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1024
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: add x0, x0, #1, lsl #12
+; CHECK-NEXT: ldg x0, [x0]
+}
+
+define i32* @get_tag4stack() {
+entry:
+; CHECK-LABEL: get_tag4stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1024
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret i32* %3
+; CHECK: mov [[T0:x[0-9]+]], sp
+; CHECK-NEXT: add x[[T1:[0-9]+]], [[T0]], #1, lsl #12
+; CHECK-NEXT: ldg x[[T1]], [x[[T1]]]
+}
+
+define i32* @get_tag5(i32* %ptr) {
+entry:
+; CHECK-LABEL: get_tag5:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 5
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0)
+  %2 = bitcast i8* %1 to i32*
+  ret i32* %2
+; CHECK: add x0, x0, #20
+; CHECK-NEXT: ldg x0, [x0]
+}
+
+define i32* @get_tag5stack() {
+entry:
+; CHECK-LABEL: get_tag5stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 5
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1)
+  %3 = bitcast i8* %2 to i32*
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret i32* %3
+; CHECK: mov [[T0:x[0-9]+]], sp
+; CHECK: add x[[T1:[0-9]+]], [[T0]], #20
+; CHECK-NEXT: ldg x[[T1]], [x[[T1]]]
+}
+
+
+; *********** __arm_mte_set_tag  *************
+define void @set_tag1(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag1:
+  %0 = bitcast i32* %tag to i8*
+  %1 = bitcast i32* %ptr to i8*
+  tail call void @llvm.aarch64.stg(i8* %0, i8* %1)
+  ret void
+; CHECK: stg x0, [x1]
+}
+
+define void @set_tag1stack(i32* %tag) {
+entry:
+; CHECK-LABEL: set_tag1stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast i32* %tag to i8*
+  %1 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %1)
+  call void @llvm.aarch64.stg(i8* %0, i8* nonnull %1)
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret void
+; CHECK: stg x0, [sp]
+}
+
+
+define void @set_tag2(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag2:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 4
+  %0 = bitcast i32* %tag to i8*
+  %1 = bitcast i32* %add.ptr to i8*
+  tail call void @llvm.aarch64.stg(i8* %0, i8* %1)
+  ret void
+; CHECK: stg x0, [x1, #16]
+}
+
+define void @set_tag2stack(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag2stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 4
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = bitcast i32* %tag to i8*
+  call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1)
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret void
+; CHECK: stg x0, [sp, #16]
+}
+
+
+
+define void @set_tag3(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag3:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1020
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = bitcast i32* %tag to i8*
+  tail call void @llvm.aarch64.stg(i8* %1, i8* %0)
+  ret void
+; CHECK: stg x0, [x1, #4080]
+}
+
+define void @set_tag3stack(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag3stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1020
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = bitcast i32* %tag to i8*
+  call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1)
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret void
+; CHECK: stg x0, [sp, #4080]
+}
+
+
+
+define void @set_tag4(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag4:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1024
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = bitcast i32* %tag to i8*
+  tail call void @llvm.aarch64.stg(i8* %1, i8* %0)
+  ret void
+; CHECK: add x[[T0:[0-9]+]], x1, #1, lsl #12
+; CHECK-NEXT: stg x0, [x[[T0]]]
+}
+
+define void @set_tag4stack(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag4stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1024
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = bitcast i32* %tag to i8*
+  call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1)
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret void
+; CHECK: add x[[T0:[0-9]+]], {{.*}}, #1, lsl #12
+; CHECK-NEXT: stg x0, [x[[T0]]]
+}
+
+
+define void @set_tag5(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag5:
+  %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 5
+  %0 = bitcast i32* %add.ptr to i8*
+  %1 = bitcast i32* %tag to i8*
+  tail call void @llvm.aarch64.stg(i8* %1, i8* %0)
+  ret void
+; CHECK: add x[[T0:[0-9]+]], x1, #20
+; CHECK-NEXT: stg x0, [x[[T0]]]
+}
+
+define void @set_tag5stack(i32* %tag, i32* %ptr) {
+entry:
+; CHECK-LABEL: set_tag5stack:
+  %s = alloca %struct.S8K, align 4
+  %0 = bitcast %struct.S8K* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0)
+  %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 5
+  %1 = bitcast i32* %arrayidx to i8*
+  %2 = bitcast i32* %tag to i8*
+  call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1)
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0)
+  ret void
+; CHECK: add x[[T0:[0-9]+]], {{.*}}, #20
+; CHECK-NEXT: stg x0, [x[[T0]]]
+}
+
+
+; *********** __arm_mte_ptrdiff  *************
+define i64 @subtract_pointers(i32* %ptra, i32* %ptrb) {
+entry:
+; CHECK-LABEL: subtract_pointers:
+  %0 = bitcast i32* %ptra to i8*
+  %1 = bitcast i32* %ptrb to i8*
+  %2 = tail call i64 @llvm.aarch64.subp(i8* %0, i8* %1)
+  ret i64 %2
+; CHECK: subp x0, x0, x1
+}
+
+declare i8* @llvm.aarch64.irg(i8*, i64)
+declare i8* @llvm.aarch64.addg(i8*, i64)
+declare i64 @llvm.aarch64.gmi(i8*, i64)
+declare i8* @llvm.aarch64.ldg(i8*, i8*)
+declare void @llvm.aarch64.stg(i8*, i8*)
+declare i64 @llvm.aarch64.subp(i8*, i8*)
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)