Index: include/llvm/IR/IntrinsicsAArch64.td =================================================================== --- include/llvm/IR/IntrinsicsAArch64.td +++ include/llvm/IR/IntrinsicsAArch64.td @@ -684,3 +684,20 @@ def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; } + +//===----------------------------------------------------------------------===// +// Memory Tagging Extensions (MTE) Intrinsics +let TargetPrefix = "aarch64" in { +def int_aarch64_irg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], + [IntrInaccessibleMemOnly]>; +def int_aarch64_addg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], + [IntrNoMem]>; +def int_aarch64_gmi : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], + [IntrNoMem]>; +def int_aarch64_ldg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], + [IntrReadMem]>; +def int_aarch64_stg : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], + [IntrWriteMem]>; +def int_aarch64_subp : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], + [IntrNoMem]>; +} Index: lib/Target/AArch64/AArch64ISelDAGToDAG.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -91,6 +91,12 @@ bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) { return SelectAddrModeIndexed7S(N, 16, Base, OffImm); } + bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) { + return SelectAddrModeIndexedBitWidth(N, true, 9, 16, Base, OffImm); + } + bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) { + return SelectAddrModeIndexedBitWidth(N, false, 6, 16, Base, OffImm); + } bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) { return SelectAddrModeIndexed(N, 1, Base, OffImm); } @@ -179,7 +185,12 @@ bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg, SDValue &Shift); bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base, - SDValue &OffImm); + SDValue &OffImm) { + return SelectAddrModeIndexedBitWidth(N, true, 7, Size, Base, OffImm); + } + bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW, + unsigned Size, SDValue &Base, + SDValue &OffImm); bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base, SDValue &OffImm); bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base, @@ -675,12 +686,13 @@ return true; } -/// SelectAddrModeIndexed7S - Select a "register plus scaled signed 7-bit +/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit /// immediate" address. The "Size" argument is the size in bytes of the memory /// reference, which determines the scale. -bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(SDValue N, unsigned Size, - SDValue &Base, - SDValue &OffImm) { +bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, + unsigned BW, unsigned Size, + SDValue &Base, + SDValue &OffImm) { SDLoc dl(N); const DataLayout &DL = CurDAG->getDataLayout(); const TargetLowering *TLI = getTargetLowering(); @@ -693,24 +705,41 @@ // As opposed to the (12-bit) Indexed addressing mode below, the 7-bit signed // selected here doesn't support labels/immediates, only base+offset. - if (CurDAG->isBaseWithConstantOffset(N)) { if (ConstantSDNode *RHS = dyn_cast(N.getOperand(1))) { - int64_t RHSC = RHS->getSExtValue(); - unsigned Scale = Log2_32(Size); - if ((RHSC & (Size - 1)) == 0 && RHSC >= -(0x40 << Scale) && - RHSC < (0x40 << Scale)) { - Base = N.getOperand(0); - if (Base.getOpcode() == ISD::FrameIndex) { - int FI = cast(Base)->getIndex(); - Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); + if (IsSignedImm) { + int64_t RHSC = RHS->getSExtValue(); + unsigned Scale = Log2_32(Size); + int64_t Range = 0x1 << (BW-1); + + if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) && + RHSC < (Range << Scale)) { + Base = N.getOperand(0); + if (Base.getOpcode() == ISD::FrameIndex) { + int FI = cast(Base)->getIndex(); + Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); + } + OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); + return true; + } + } else { + // unsigned Immediate + uint64_t RHSC = RHS->getZExtValue(); + unsigned Scale = Log2_32(Size); + uint64_t Range = 0x1 << BW; + + if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) { + Base = N.getOperand(0); + if (Base.getOpcode() == ISD::FrameIndex) { + int FI = cast(Base)->getIndex(); + Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); + } + OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); + return true; } - OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); - return true; } } } - // Base only. The address will be materialized into a register before // the memory is accessed. // add x0, Xbase, #offset Index: lib/Target/AArch64/AArch64InstrFormats.td =================================================================== --- lib/Target/AArch64/AArch64InstrFormats.td +++ lib/Target/AArch64/AArch64InstrFormats.td @@ -355,6 +355,9 @@ def am_indexed7s64 : ComplexPattern; def am_indexed7s128 : ComplexPattern; +def am_indexedu6s128 : ComplexPattern; +def am_indexeds9s128 : ComplexPattern; + // uimm5sN predicate - True if the immediate is a multiple of N in the range // [0 * N, 32 * N]. def UImm5s2Operand : UImmScaledMemoryIndexed<5, 2>; @@ -2339,7 +2342,7 @@ } class SUBP - : BaseTwoOperand<0b0000, GPR64, asm_instr, null_frag, GPR64sp, GPR64sp> { + : BaseTwoOperand<0b0000, GPR64, asm_instr, OpNode, GPR64sp, GPR64sp> { let Inst{31} = 1; let Inst{29} = setsFlags; } Index: lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.cpp +++ lib/Target/AArch64/AArch64InstrInfo.cpp @@ -1768,7 +1768,11 @@ case AArch64::LDNPSi: case AArch64::STNPWi: case AArch64::STNPSi: + case AArch64::LDG: return 3; + case AArch64::ADDG: + case AArch64::STGOffset: + return 2; } } @@ -2143,6 +2147,18 @@ MinOffset = 0; MaxOffset = 4095; break; + case AArch64::ADDG: + Scale = 16; + Width = 0; + MinOffset = 0; + MaxOffset = 63; + break; + case AArch64::LDG: + case AArch64::STGOffset: + Scale = Width = 16; + MinOffset = -256; + MaxOffset = 255; + break; } return true; Index: lib/Target/AArch64/AArch64InstrInfo.td =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.td +++ lib/Target/AArch64/AArch64InstrInfo.td @@ -1233,11 +1233,11 @@ // v8.5 Memory Tagging Extension let Predicates = [HasMTE] in { -def IRG : BaseTwoOperand<0b0100, GPR64sp, "irg", null_frag, GPR64sp, GPR64>, +def IRG : BaseTwoOperand<0b0100, GPR64sp, "irg", int_aarch64_irg, GPR64sp, GPR64>, Sched<[]>{ let Inst{31} = 1; } -def GMI : BaseTwoOperand<0b0101, GPR64, "gmi", null_frag, GPR64sp>, Sched<[]>{ +def GMI : BaseTwoOperand<0b0101, GPR64, "gmi", int_aarch64_gmi, GPR64sp>, Sched<[]>{ let Inst{31} = 1; let isNotDuplicable = 1; } @@ -1246,7 +1246,7 @@ def : InstAlias<"irg $dst, $src", (IRG GPR64sp:$dst, GPR64sp:$src, XZR), 1>; -def SUBP : SUBP<0, "subp", null_frag>, Sched<[]>; +def SUBP : SUBP<0, "subp", int_aarch64_subp>, Sched<[]>; def SUBPS : SUBP<1, "subps", null_frag>, Sched<[]>{ let Defs = [NZCV]; } @@ -1254,13 +1254,18 @@ def : InstAlias<"cmpp $lhs, $rhs", (SUBPS XZR, GPR64sp:$lhs, GPR64sp:$rhs), 0>; def LDG : MemTagLoad<"ldg", "\t$Rt, [$Rn, $offset]">; + +def : Pat<(int_aarch64_addg (am_indexedu6s128 GPR64sp:$Rn, uimm6s16:$imm6), imm0_15:$imm4), + (ADDG GPR64sp:$Rn, imm0_63:$imm6, imm0_15:$imm4)>; +def : Pat<(int_aarch64_ldg GPR64:$Rt, (am_indexeds9s128 GPR64sp:$Rn, simm9s16:$offset)), + (LDG GPR64:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; + def : InstAlias<"ldg $Rt, [$Rn]", (LDG GPR64:$Rt, GPR64sp:$Rn, 0), 1>; def LDGM : MemTagVector<1, "ldgm", "\t$Rt, [$Rn]", (outs GPR64:$Rt), (ins GPR64sp:$Rn)>; def STGM : MemTagVector<0, "stgm", "\t$Rt, [$Rn]", (outs), (ins GPR64:$Rt, GPR64sp:$Rn)>; - def STZGM : MemTagVector<0, "stzgm", "\t$Rt, [$Rn]", (outs), (ins GPR64:$Rt, GPR64sp:$Rn)> { let Inst{23} = 0; @@ -1275,6 +1280,9 @@ def STGPpre : StorePairPreIdx <0b01, 0, GPR64z, simm7s16, "stgp">; def STGPpost : StorePairPostIdx<0b01, 0, GPR64z, simm7s16, "stgp">; +def : Pat<(int_aarch64_stg GPR64:$Rt, (am_indexeds9s128 GPR64sp:$Rn, simm9s16:$offset)), + (STGOffset GPR64:$Rt, GPR64sp:$Rn, simm9s16:$offset)>; + } // Predicates = [HasMTE] //===----------------------------------------------------------------------===// Index: test/CodeGen/AArch64/arm64-mte.ll =================================================================== --- /dev/null +++ test/CodeGen/AArch64/arm64-mte.ll @@ -0,0 +1,497 @@ +; RUN: llc < %s -mtriple=arm64-eabi -mattr=+mte | FileCheck %s + +; test create_tag +define i32* @create_tag(i32* %ptr, i32 %m) { +entry: +; CHECK-LABEL: create_tag: + %0 = bitcast i32* %ptr to i8* + %1 = zext i32 %m to i64 + %2 = tail call i8* @llvm.aarch64.irg(i8* %0, i64 %1) + %3 = bitcast i8* %2 to i32* + ret i32* %3 +;CHECK: irg x0, x0, {{x[0-9]+}} +} + +; *********** __arm_mte_increment_tag ************* +; test increment_tag1 +define i32* @increment_tag1(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag1: + %0 = bitcast i32* %ptr to i8* + %1 = tail call i8* @llvm.aarch64.addg(i8* %0, i64 7) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: addg x0, x0, #0, #7 +} + +%struct.S2K = type { [512 x i32] } +define i32* @increment_tag1stack(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag1stack: + %s = alloca %struct.S2K, align 4 + %0 = bitcast %struct.S2K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0) + %1 = call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7) + %2 = bitcast i8* %1 to i32* + call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0) + ret i32* %2 +; CHECK: addg x0, sp, #0, #7 +} + + +define i32* @increment_tag2(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag2: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 4 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: addg x0, x0, #16, #7 +} + +define i32* @increment_tag2stack(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag2stack: + %s = alloca %struct.S2K, align 4 + %0 = bitcast %struct.S2K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 4 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0) + ret i32* %3 +; CHECK: addg x0, sp, #16, #7 +} + +define i32* @increment_tag3(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag3: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 252 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: addg x0, x0, #1008, #7 +} + +define i32* @increment_tag3stack(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag3stack: + %s = alloca %struct.S2K, align 4 + %0 = bitcast %struct.S2K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 252 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0) + ret i32* %3 +; CHECK: addg x0, sp, #1008, #7 +} + + +define i32* @increment_tag4(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag4: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 256 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: add [[T0:x[0-9]+]], x0, #1024 +; CHECK-NEXT: addg x0, [[T0]], #0, #7 +} + +define i32* @increment_tag4stack(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag4stack: + %s = alloca %struct.S2K, align 4 + %0 = bitcast %struct.S2K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 256 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0) + ret i32* %3 +; CHECK: add [[T0:x[0-9]+]], {{.*}}, #1024 +; CHECK-NEXT: addg x0, [[T0]], #0, #7 +} + + +define i32* @increment_tag5(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag5: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 5 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.addg(i8* nonnull %0, i64 7) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: add [[T0:x[0-9]+]], x0, #20 +; CHECK-NEXT: addg x0, [[T0]], #0, #7 +} + +define i32* @increment_tag5stack(i32* %ptr) { +entry: +; CHECK-LABEL: increment_tag5stack: + %s = alloca %struct.S2K, align 4 + %0 = bitcast %struct.S2K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 2048, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S2K, %struct.S2K* %s, i64 0, i32 0, i64 5 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.addg(i8* nonnull %1, i64 7) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 2048, i8* nonnull %0) + ret i32* %3 +; CHECK: add [[T0:x[0-9]+]], {{.*}}, #20 +; CHECK-NEXT: addg x0, [[T0]], #0, #7 +} + + +; *********** __arm_mte_exclude_tag ************* +; test exclude_tag +define i32 @exclude_tag(i32* %ptr, i32 %m) local_unnamed_addr #0 { +entry: +;CHECK-LABEL: exclude_tag: + %0 = zext i32 %m to i64 + %1 = bitcast i32* %ptr to i8* + %2 = tail call i64 @llvm.aarch64.gmi(i8* %1, i64 %0) + %conv = trunc i64 %2 to i32 + ret i32 %conv +; CHECK: gmi x0, x0, {{x[0-9]+}} +} + + +; *********** __arm_mte_get_tag ************* +%struct.S8K = type { [2048 x i32] } +define i32* @get_tag1(i32* %ptr) { +entry: +; CHECK-LABEL: get_tag1: + %0 = bitcast i32* %ptr to i8* + %1 = tail call i8* @llvm.aarch64.ldg(i8* %0, i8* %0) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK ldg x0, [x0] +} + +define i32* @get_tag1_two_parm(i32* %ret_ptr, i32* %ptr) { +entry: +; CHECK-LABEL: get_tag1_two_parm: + %0 = bitcast i32* %ret_ptr to i8* + %1 = bitcast i32* %ptr to i8* + %2 = tail call i8* @llvm.aarch64.ldg(i8* %0, i8* %1) + %3 = bitcast i8* %2 to i32* + ret i32* %3 +; CHECK ldg x0, [x1] +} + +define i32* @get_tag1stack() { +entry: +; CHECK-LABEL: get_tag1stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %1 = call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0) + %2 = bitcast i8* %1 to i32* + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret i32* %2 +; CHECK: mov [[T0:x[0-9]+]], sp +; CHECK: ldg [[T0]], [sp] +} + +define i32* @get_tag1stack_two_param(i32* %ret_ptr) { +entry: +; CHECK-LABEL: get_tag1stack_two_param: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + %1 = bitcast i32* %ret_ptr to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %0) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret i32* %3 +; CHECK-NOT: mov {{.*}}, sp +; CHECK: ldg x0, [sp] +} + + +define i32* @get_tag2(i32* %ptr) { +entry: +; CHECK-LABEL: get_tag2: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 4 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: add [[T0:x[0-9]+]], x0, #16 +; CHECK: ldg [[T0]], [x0, #16] +} + +define i32* @get_tag2stack() { +entry: +; CHECK-LABEL: get_tag2stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 4 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret i32* %3 +; CHECK: mov [[T0:x[0-9]+]], sp +; CHECK: add x0, [[T0]], #16 +; CHECK: ldg x0, [sp, #16] +} + + +define i32* @get_tag3(i32* %ptr) { +entry: +; CHECK-LABEL: get_tag3: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1020 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: add [[T0:x[0-8]+]], x0, #4080 +; CHECK: ldg [[T0]], [x0, #4080] +} + +define i32* @get_tag3stack() { +entry: +; CHECK-LABEL: get_tag3stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1020 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret i32* %3 +; CHECK: mov [[T0:x[0-9]+]], sp +; CHECK: add x0, [[T0]], #4080 +; CHECK: ldg x0, [sp, #4080] +} + + +define i32* @get_tag4(i32* %ptr) { +entry: +; CHECK-LABEL: get_tag4: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1024 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: add x0, x0, #1, lsl #12 +; CHECK-NEXT: ldg x0, [x0] +} + +define i32* @get_tag4stack() { +entry: +; CHECK-LABEL: get_tag4stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1024 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret i32* %3 +; CHECK: mov [[T0:x[0-9]+]], sp +; CHECK-NEXT: add x[[T1:[0-9]+]], [[T0]], #1, lsl #12 +; CHECK-NEXT: ldg x[[T1]], [x[[T1]]] +} + +define i32* @get_tag5(i32* %ptr) { +entry: +; CHECK-LABEL: get_tag5: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 5 + %0 = bitcast i32* %add.ptr to i8* + %1 = tail call i8* @llvm.aarch64.ldg(i8* nonnull %0, i8* nonnull %0) + %2 = bitcast i8* %1 to i32* + ret i32* %2 +; CHECK: add x0, x0, #20 +; CHECK-NEXT: ldg x0, [x0] +} + +define i32* @get_tag5stack() { +entry: +; CHECK-LABEL: get_tag5stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 5 + %1 = bitcast i32* %arrayidx to i8* + %2 = call i8* @llvm.aarch64.ldg(i8* nonnull %1, i8* nonnull %1) + %3 = bitcast i8* %2 to i32* + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret i32* %3 +; CHECK: mov [[T0:x[0-9]+]], sp +; CHECK: add x[[T1:[0-9]+]], [[T0]], #20 +; CHECK-NEXT: ldg x[[T1]], [x[[T1]]] +} + + +; *********** __arm_mte_set_tag ************* +define void @set_tag1(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag1: + %0 = bitcast i32* %tag to i8* + %1 = bitcast i32* %ptr to i8* + tail call void @llvm.aarch64.stg(i8* %0, i8* %1) + ret void +; CHECK: stg x0, [x1] +} + +define void @set_tag1stack(i32* %tag) { +entry: +; CHECK-LABEL: set_tag1stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast i32* %tag to i8* + %1 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %1) + call void @llvm.aarch64.stg(i8* %0, i8* nonnull %1) + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret void +; CHECK: stg x0, [sp] +} + + +define void @set_tag2(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag2: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 4 + %0 = bitcast i32* %tag to i8* + %1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.aarch64.stg(i8* %0, i8* %1) + ret void +; CHECK: stg x0, [x1, #16] +} + +define void @set_tag2stack(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag2stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 4 + %1 = bitcast i32* %arrayidx to i8* + %2 = bitcast i32* %tag to i8* + call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1) + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret void +; CHECK: stg x0, [sp, #16] +} + + + +define void @set_tag3(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag3: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1020 + %0 = bitcast i32* %add.ptr to i8* + %1 = bitcast i32* %tag to i8* + tail call void @llvm.aarch64.stg(i8* %1, i8* %0) + ret void +; CHECK: stg x0, [x1, #4080] +} + +define void @set_tag3stack(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag3stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1020 + %1 = bitcast i32* %arrayidx to i8* + %2 = bitcast i32* %tag to i8* + call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1) + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret void +; CHECK: stg x0, [sp, #4080] +} + + + +define void @set_tag4(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag4: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 1024 + %0 = bitcast i32* %add.ptr to i8* + %1 = bitcast i32* %tag to i8* + tail call void @llvm.aarch64.stg(i8* %1, i8* %0) + ret void +; CHECK: add x[[T0:[0-9]+]], x1, #1, lsl #12 +; CHECK-NEXT: stg x0, [x[[T0]]] +} + +define void @set_tag4stack(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag4stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 1024 + %1 = bitcast i32* %arrayidx to i8* + %2 = bitcast i32* %tag to i8* + call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1) + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret void +; CHECK: add x[[T0:[0-9]+]], {{.*}}, #1, lsl #12 +; CHECK-NEXT: stg x0, [x[[T0]]] +} + + +define void @set_tag5(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag5: + %add.ptr = getelementptr inbounds i32, i32* %ptr, i64 5 + %0 = bitcast i32* %add.ptr to i8* + %1 = bitcast i32* %tag to i8* + tail call void @llvm.aarch64.stg(i8* %1, i8* %0) + ret void +; CHECK: add x[[T0:[0-9]+]], x1, #20 +; CHECK-NEXT: stg x0, [x[[T0]]] +} + +define void @set_tag5stack(i32* %tag, i32* %ptr) { +entry: +; CHECK-LABEL: set_tag5stack: + %s = alloca %struct.S8K, align 4 + %0 = bitcast %struct.S8K* %s to i8* + call void @llvm.lifetime.start.p0i8(i64 8192, i8* nonnull %0) + %arrayidx = getelementptr inbounds %struct.S8K, %struct.S8K* %s, i64 0, i32 0, i64 5 + %1 = bitcast i32* %arrayidx to i8* + %2 = bitcast i32* %tag to i8* + call void @llvm.aarch64.stg(i8* %2, i8* nonnull %1) + call void @llvm.lifetime.end.p0i8(i64 8192, i8* nonnull %0) + ret void +; CHECK: add x[[T0:[0-9]+]], {{.*}}, #20 +; CHECK-NEXT: stg x0, [x[[T0]]] +} + + +; *********** __arm_mte_ptrdiff ************* +define i64 @subtract_pointers(i32* %ptra, i32* %ptrb) { +entry: +; CHECK-LABEL: subtract_pointers: + %0 = bitcast i32* %ptra to i8* + %1 = bitcast i32* %ptrb to i8* + %2 = tail call i64 @llvm.aarch64.subp(i8* %0, i8* %1) + ret i64 %2 +; CHECK: subp x0, x0, x1 +} + +declare i8* @llvm.aarch64.irg(i8*, i64) +declare i8* @llvm.aarch64.addg(i8*, i64) +declare i64 @llvm.aarch64.gmi(i8*, i64) +declare i8* @llvm.aarch64.ldg(i8*, i8*) +declare void @llvm.aarch64.stg(i8*, i8*) +declare i64 @llvm.aarch64.subp(i8*, i8*) + +declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) +declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)