[OpaquePtr][AArch64] Use elementtype on ldxr/stxr

[aeubanks]

Summary

Includes verifier changes checking the elementtype, clang codegen
changes to emit the elementtype, and ISel changes using the elementtype.

[aeubanks]

update

[nikic]

An alternative I suggested on https://github.com/llvm/llvm-project/issues/51165 was to overload these intrinsics by result/value type. I think that would be cleaner design-wise, but it also seems harder to implement (requires custom ISD nodes), so I'm okay with the elementtype approach.

llvm/lib/IR/Verifier.cpp

5541

This is checked generically for all elementtype attributes. You can also merge this into the previous switch case.

llvm/test/Verifier/AArch64/lit.local.cfg

2 ↗

(On Diff #411274)

This is unnecessary, target intrinsics are available even if the target isn't.

[aeubanks]

address comments

[nikic]

LGTM. You probably want to add these intrinsics to the auto-upgrade code in BitcodeReader::propagateAttributeTypes() as well.

[dmgreen]

Seems OK. Thanks for the patch.

Do opaque pointer variants (like i32 @llvm.aarch64.stxr.p0(i64 1, ptr elementtype(i64) %ptr.0)) get tested automatically from the existing tests once -opaque-pointers is the default?

[aeubanks]

update

[aeubanks]

In D120527#3372092, @dmgreen wrote:

Seems OK. Thanks for the patch.

Do opaque pointer variants (like i32 @llvm.aarch64.stxr.p0(i64 1, ptr elementtype(i64) %ptr.0)) get tested automatically from the existing tests once -opaque-pointers is the default?

yup

In D120527#3371927, @nikic wrote:

LGTM. You probably want to add these intrinsics to the auto-upgrade code in BitcodeReader::propagateAttributeTypes() as well.

done

[nikic]

@aeubanks Do you plan to take care of the corresponding arm intrinsics as well?

[aeubanks]

In D120527#3381835, @nikic wrote:

@aeubanks Do you plan to take care of the corresponding arm intrinsics as well?

yes I'll do those

clang/lib/CodeGen/CGBuiltin.cpp

if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
     Val = Builder.CreateOr(Val, Val1);
     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
   } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
              BuiltinID == AArch64::BI__builtin_arm_ldaex) {
     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
 
     QualType Ty = E->getType();
     llvm::Type *RealResTy = ConvertType(Ty);
-    llvm::Type *PtrTy = llvm::IntegerType::get(
-        getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
+    llvm::Type *IntTy =
+        llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
+    llvm::Type *PtrTy = IntTy->getPointerTo();
     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
 
     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
                                        ? Intrinsic::aarch64_ldaxr
                                        : Intrinsic::aarch64_ldxr,
                                    PtrTy);
-    Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
+    CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
+    Val->addParamAttr(
+        0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
 
     if (RealResTy->isPointerTy())
       return Builder.CreateIntToPtr(Val, RealResTy);
 
     llvm::Type *IntResTy = llvm::IntegerType::get(
         getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
-    Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
-    return Builder.CreateBitCast(Val, RealResTy);
+    return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
+                                 RealResTy);
   }
 
   if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
        BuiltinID == AArch64::BI__builtin_arm_stlex) &&
       getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
                                        ? Intrinsic::aarch64_stlxp
                                        : Intrinsic::aarch64_stxp);
else {
       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
     }
 
     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
                                        ? Intrinsic::aarch64_stlxr
                                        : Intrinsic::aarch64_stxr,
                                    StoreAddr->getType());
-    return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
+    CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
+    CI->addParamAttr(
+        1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
+    return CI;
   }
 
   if (BuiltinID == AArch64::BI__getReg) {
     Expr::EvalResult Result;
     if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
       llvm_unreachable("Sema will ensure that the parameter is constant");
 
     llvm::APSInt Value = Result.Val.getInt();

clang/test/CodeGen/arm_acle.c

 // AArch32:       __swp.exit:
 // AArch32-NEXT:    ret void
 //
 // AArch64-LABEL: @test_swp(
 // AArch64-NEXT:  entry:
 // AArch64-NEXT:    [[TMP0:%.*]] = bitcast i8* [[P:%.*]] to i32*
 // AArch64-NEXT:    br label [[DO_BODY_I:%.*]]
 // AArch64:       do.body.i:
-// AArch64-NEXT:    [[LDXR_I:%.*]] = call i64 @llvm.aarch64.ldxr.p0i32(i32* [[TMP0]]) [[ATTR3]]
+// AArch64-NEXT:    [[LDXR_I:%.*]] = call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32) [[TMP0]]) [[ATTR3]]
 // AArch64-NEXT:    [[TMP1:%.*]] = trunc i64 [[LDXR_I]] to i32
 // AArch64-NEXT:    [[TMP2:%.*]] = zext i32 [[X:%.*]] to i64
-// AArch64-NEXT:    [[STXR_I:%.*]] = call i32 @llvm.aarch64.stxr.p0i32(i64 [[TMP2]], i32* [[TMP0]]) [[ATTR3]]
+// AArch64-NEXT:    [[STXR_I:%.*]] = call i32 @llvm.aarch64.stxr.p0i32(i64 [[TMP2]], i32* elementtype(i32) [[TMP0]]) [[ATTR3]]
 // AArch64-NEXT:    [[TOBOOL_I:%.*]] = icmp ne i32 [[STXR_I]], 0
 // AArch64-NEXT:    br i1 [[TOBOOL_I]], label [[DO_BODY_I]], label [[__SWP_EXIT:%.*]], [[LOOP6:!llvm.loop !.*]]
 // AArch64:       __swp.exit:
 // AArch64-NEXT:    ret void
 //
 void test_swp(uint32_t x, volatile void *p) {
   __swp(x, p);
 }

clang/test/CodeGen/builtins-arm-exclusive.c

 // RUN: %clang_cc1 -Wall -Werror -triple thumbv8-linux-gnueabi -fno-signed-char -disable-O0-optnone -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
 // RUN: %clang_cc1 -Wall -Werror -triple arm64-apple-ios7.0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s --check-prefix=CHECK-ARM64
 
 struct Simple {
   char a, b;
 };
 
 int test_ldrex(char *addr, long long *addr64, float *addrfloat) {
 // CHECK-LABEL: @test_ldrex
 // CHECK-ARM64-LABEL: @test_ldrex
   int sum = 0;
   sum += __builtin_arm_ldrex(addr);
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %addr)
 // CHECK: trunc i32 [[INTRES]] to i8
 
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i8(i8* elementtype(i8) %addr)
 // CHECK-ARM64: trunc i64 [[INTRES]] to i8
 
   sum += __builtin_arm_ldrex((short *)addr);
 // CHECK: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* [[ADDR16]])
 // CHECK: trunc i32 [[INTRES]] to i16
 
 // CHECK-ARM64: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i16(i16* [[ADDR16]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i16(i16* elementtype(i16) [[ADDR16]])
 // CHECK-ARM64: trunc i64 [[INTRES]] to i16
 
   sum += __builtin_arm_ldrex((int *)addr);
 // CHECK: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
 // CHECK: call i32 @llvm.arm.ldrex.p0i32(i32* [[ADDR32]])
 
 // CHECK-ARM64: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i32(i32* [[ADDR32]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32) [[ADDR32]])
 // CHECK-ARM64: trunc i64 [[INTRES]] to i32
 
   sum += __builtin_arm_ldrex((long long *)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to i64*
 // CHECK: [[TMP5:%.*]] = bitcast i64* [[TMP4]] to i8*
 // CHECK: call { i32, i32 } @llvm.arm.ldrexd(i8* [[TMP5]])
 
 // CHECK-ARM64: [[ADDR64:%.*]] = bitcast i8* %addr to i64*
-// CHECK-ARM64: call i64 @llvm.aarch64.ldxr.p0i64(i64* [[ADDR64]])
+// CHECK-ARM64: call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) [[ADDR64]])
 
   sum += __builtin_arm_ldrex(addr64);
 // CHECK: [[ADDR64_AS8:%.*]] = bitcast i64* %addr64 to i8*
 // CHECK: call { i32, i32 } @llvm.arm.ldrexd(i8* [[ADDR64_AS8]])
 
-// CHECK-ARM64: call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr64)
+// CHECK-ARM64: call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) %addr64)
 
   sum += __builtin_arm_ldrex(addrfloat);
 // CHECK: [[INTADDR:%.*]] = bitcast float* %addrfloat to i32*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* [[INTADDR]])
 // CHECK: bitcast i32 [[INTRES]] to float
 
 // CHECK-ARM64: [[INTADDR:%.*]] = bitcast float* %addrfloat to i32*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i32(i32* [[INTADDR]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32) [[INTADDR]])
 // CHECK-ARM64: [[TRUNCRES:%.*]] = trunc i64 [[INTRES]] to i32
 // CHECK-ARM64: bitcast i32 [[TRUNCRES]] to float
 
   sum += __builtin_arm_ldrex((double *)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i8*
 // CHECK: [[STRUCTRES:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[TMP5]])
 // CHECK: [[RESHI:%.*]] = extractvalue { i32, i32 } [[STRUCTRES]], 1
 // CHECK: [[RESLO:%.*]] = extractvalue { i32, i32 } [[STRUCTRES]], 0
 // CHECK: [[RESHI64:%.*]] = zext i32 [[RESHI]] to i64
 // CHECK: [[RESLO64:%.*]] = zext i32 [[RESLO]] to i64
 // CHECK: [[RESHIHI:%.*]] = shl nuw i64 [[RESHI64]], 32
 // CHECK: [[INTRES:%.*]] = or i64 [[RESHIHI]], [[RESLO64]]
 // CHECK: bitcast i64 [[INTRES]] to double
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i64*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i64(i64* [[TMP5]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) [[TMP5]])
 // CHECK-ARM64: bitcast i64 [[INTRES]] to double
 
   sum += *__builtin_arm_ldrex((int **)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to i32**
 // CHECK: [[TMP5:%.*]] = bitcast i32** [[TMP4]] to i32*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* [[TMP5]])
 // CHECK: inttoptr i32 [[INTRES]] to i32*
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to i32**
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast i32** [[TMP4]] to i64*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i64(i64* [[TMP5]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) [[TMP5]])
 // CHECK-ARM64: inttoptr i64 [[INTRES]] to i32*
 
   sum += __builtin_arm_ldrex((struct Simple **)addr)->a;
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i32*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* [[TMP5]])
 // CHECK: inttoptr i32 [[INTRES]] to %struct.Simple*
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i64*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i64(i64* [[TMP5]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) [[TMP5]])
 // CHECK-ARM64: inttoptr i64 [[INTRES]] to %struct.Simple*
   return sum;
 }
 
 int test_ldaex(char *addr, long long *addr64, float *addrfloat) {
 // CHECK-LABEL: @test_ldaex
 // CHECK-ARM64-LABEL: @test_ldaex
   int sum = 0;
   sum += __builtin_arm_ldaex(addr);
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldaex.p0i8(i8* %addr)
 // CHECK: trunc i32 [[INTRES]] to i8
 
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i8(i8* elementtype(i8) %addr)
 // CHECK-ARM64: trunc i64 [[INTRES]] to i8
 
   sum += __builtin_arm_ldaex((short *)addr);
 // CHECK: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* [[ADDR16]])
 // CHECK: trunc i32 [[INTRES]] to i16
 
 // CHECK-ARM64: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i16(i16* [[ADDR16]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i16(i16* elementtype(i16) [[ADDR16]])
 // CHECK-ARM64: [[TRUNCRES:%.*]] = trunc i64 [[INTRES]] to i16
 
   sum += __builtin_arm_ldaex((int *)addr);
 // CHECK: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
 // CHECK:  call i32 @llvm.arm.ldaex.p0i32(i32* [[ADDR32]])
 
 // CHECK-ARM64: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i32(i32* [[ADDR32]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i32(i32* elementtype(i32) [[ADDR32]])
 // CHECK-ARM64: trunc i64 [[INTRES]] to i32
 
   sum += __builtin_arm_ldaex((long long *)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to i64*
 // CHECK: [[TMP5:%.*]] = bitcast i64* [[TMP4]] to i8*
 // CHECK: call { i32, i32 } @llvm.arm.ldaexd(i8* [[TMP5]])
 
 // CHECK-ARM64: [[ADDR64:%.*]] = bitcast i8* %addr to i64*
-// CHECK-ARM64: call i64 @llvm.aarch64.ldaxr.p0i64(i64* [[ADDR64]])
+// CHECK-ARM64: call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) [[ADDR64]])
 
   sum += __builtin_arm_ldaex(addr64);
 // CHECK: [[ADDR64_AS8:%.*]] = bitcast i64* %addr64 to i8*
 // CHECK: call { i32, i32 } @llvm.arm.ldaexd(i8* [[ADDR64_AS8]])
 
-// CHECK-ARM64: call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr64)
+// CHECK-ARM64: call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) %addr64)
 
   sum += __builtin_arm_ldaex(addrfloat);
 // CHECK: [[INTADDR:%.*]] = bitcast float* %addrfloat to i32*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* [[INTADDR]])
 // CHECK: bitcast i32 [[INTRES]] to float
 
 // CHECK-ARM64: [[INTADDR:%.*]] = bitcast float* %addrfloat to i32*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i32(i32* [[INTADDR]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i32(i32* elementtype(i32) [[INTADDR]])
 // CHECK-ARM64: [[TRUNCRES:%.*]] = trunc i64 [[INTRES]] to i32
 // CHECK-ARM64: bitcast i32 [[TRUNCRES]] to float
 
   sum += __builtin_arm_ldaex((double *)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i8*
 // CHECK: [[STRUCTRES:%.*]] = call { i32, i32 } @llvm.arm.ldaexd(i8* [[TMP5]])
 // CHECK: [[RESHI:%.*]] = extractvalue { i32, i32 } [[STRUCTRES]], 1
 // CHECK: [[RESLO:%.*]] = extractvalue { i32, i32 } [[STRUCTRES]], 0
 // CHECK: [[RESHI64:%.*]] = zext i32 [[RESHI]] to i64
 // CHECK: [[RESLO64:%.*]] = zext i32 [[RESLO]] to i64
 // CHECK: [[RESHIHI:%.*]] = shl nuw i64 [[RESHI64]], 32
 // CHECK: [[INTRES:%.*]] = or i64 [[RESHIHI]], [[RESLO64]]
 // CHECK: bitcast i64 [[INTRES]] to double
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i64*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* [[TMP5]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) [[TMP5]])
 // CHECK-ARM64: bitcast i64 [[INTRES]] to double
 
   sum += *__builtin_arm_ldaex((int **)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to i32**
 // CHECK: [[TMP5:%.*]] = bitcast i32** [[TMP4]] to i32*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* [[TMP5]])
 // CHECK: inttoptr i32 [[INTRES]] to i32*
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to i32**
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast i32** [[TMP4]] to i64*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* [[TMP5]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) [[TMP5]])
 // CHECK-ARM64: inttoptr i64 [[INTRES]] to i32*
 
   sum += __builtin_arm_ldaex((struct Simple **)addr)->a;
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i32*
 // CHECK: [[INTRES:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* [[TMP5]])
 // CHECK: inttoptr i32 [[INTRES]] to %struct.Simple*
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i64*
-// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* [[TMP5]])
+// CHECK-ARM64: [[INTRES:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) [[TMP5]])
 // CHECK-ARM64: inttoptr i64 [[INTRES]] to %struct.Simple*
   return sum;
 }
 
 int test_strex(char *addr) {
 // CHECK-LABEL: @test_strex
 // CHECK-ARM64-LABEL: @test_strex
   int res = 0;
   struct Simple var = {0};
   res |= __builtin_arm_strex(4, addr);
 // CHECK: call i32 @llvm.arm.strex.p0i8(i32 4, i8* %addr)
 
-// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i8(i64 4, i8* %addr)
+// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i8(i64 4, i8* elementtype(i8) %addr)
 
   res |= __builtin_arm_strex(42, (short *)addr);
 // CHECK: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
 // CHECK:  call i32 @llvm.arm.strex.p0i16(i32 42, i16* [[ADDR16]])
 
 // CHECK-ARM64: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
-// CHECK-ARM64:  call i32 @llvm.aarch64.stxr.p0i16(i64 42, i16* [[ADDR16]])
+// CHECK-ARM64:  call i32 @llvm.aarch64.stxr.p0i16(i64 42, i16* elementtype(i16) [[ADDR16]])
 
   res |= __builtin_arm_strex(42, (int *)addr);
 // CHECK: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
 // CHECK: call i32 @llvm.arm.strex.p0i32(i32 42, i32* [[ADDR32]])
 
 // CHECK-ARM64: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
-// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i32(i64 42, i32* [[ADDR32]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i32(i64 42, i32* elementtype(i32) [[ADDR32]])
 
   res |= __builtin_arm_strex(42, (long long *)addr);
 // CHECK: store i64 42, i64* [[TMP:%.*]], align 8
 // CHECK: [[LOHI_ADDR:%.*]] = bitcast i64* [[TMP]] to { i32, i32 }*
 // CHECK: [[LOHI:%.*]] = load { i32, i32 }, { i32, i32 }* [[LOHI_ADDR]]
 // CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
 // CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to i64*
 // CHECK: [[TMP5:%.*]] = bitcast i64* [[TMP4]] to i8*
 // CHECK: call i32 @llvm.arm.strexd(i32 [[LO]], i32 [[HI]], i8* [[TMP5]])
 
 // CHECK-ARM64: [[ADDR64:%.*]] = bitcast i8* %addr to i64*
-// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i64(i64 42, i64* [[ADDR64]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i64(i64 42, i64* elementtype(i64) [[ADDR64]])
 
   res |= __builtin_arm_strex(2.71828f, (float *)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to float*
 // CHECK: [[TMP5:%.*]] = bitcast float* [[TMP4]] to i32*
 // CHECK: call i32 @llvm.arm.strex.p0i32(i32 1076754509, i32* [[TMP5]])
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to float*
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast float* [[TMP4]] to i32*
-// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i32(i64 1076754509, i32* [[TMP5]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i32(i64 1076754509, i32* elementtype(i32) [[TMP5]])
 
   res |= __builtin_arm_strex(3.14159, (double *)addr);
 // CHECK: store double 3.141590e+00, double* [[TMP:%.*]], align 8
 // CHECK: [[LOHI_ADDR:%.*]] = bitcast double* [[TMP]] to { i32, i32 }*
 // CHECK: [[LOHI:%.*]] = load { i32, i32 }, { i32, i32 }* [[LOHI_ADDR]]
 // CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
 // CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i8*
 // CHECK: call i32 @llvm.arm.strexd(i32 [[LO]], i32 [[HI]], i8* [[TMP5]])
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i64*
-// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i64(i64 4614256650576692846, i64* [[TMP5]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i64(i64 4614256650576692846, i64* elementtype(i64) [[TMP5]])
 
   res |= __builtin_arm_strex(&var, (struct Simple **)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i32*
 // CHECK: [[INTVAL:%.*]] = ptrtoint %struct.Simple* %var to i32
 // CHECK: call i32 @llvm.arm.strex.p0i32(i32 [[INTVAL]], i32* [[TMP5]])
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i64*
 // CHECK-ARM64: [[INTVAL:%.*]] = ptrtoint %struct.Simple* %var to i64
-// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i64(i64 [[INTVAL]], i64* [[TMP5]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stxr.p0i64(i64 [[INTVAL]], i64* elementtype(i64) [[TMP5]])
 
   return res;
 }
 
 int test_stlex(char *addr) {
 // CHECK-LABEL: @test_stlex
 // CHECK-ARM64-LABEL: @test_stlex
   int res = 0;
   struct Simple var = {0};
   res |= __builtin_arm_stlex(4, addr);
 // CHECK: call i32 @llvm.arm.stlex.p0i8(i32 4, i8* %addr)
 
-// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i8(i64 4, i8* %addr)
+// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i8(i64 4, i8* elementtype(i8) %addr)
 
   res |= __builtin_arm_stlex(42, (short *)addr);
 // CHECK: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
 // CHECK:  call i32 @llvm.arm.stlex.p0i16(i32 42, i16* [[ADDR16]])
 
 // CHECK-ARM64: [[ADDR16:%.*]] = bitcast i8* %addr to i16*
-// CHECK-ARM64:  call i32 @llvm.aarch64.stlxr.p0i16(i64 42, i16* [[ADDR16]])
+// CHECK-ARM64:  call i32 @llvm.aarch64.stlxr.p0i16(i64 42, i16* elementtype(i16) [[ADDR16]])
 
   res |= __builtin_arm_stlex(42, (int *)addr);
 // CHECK: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
 // CHECK: call i32 @llvm.arm.stlex.p0i32(i32 42, i32* [[ADDR32]])
 
 // CHECK-ARM64: [[ADDR32:%.*]] = bitcast i8* %addr to i32*
-// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i32(i64 42, i32* [[ADDR32]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i32(i64 42, i32* elementtype(i32) [[ADDR32]])
 
   res |= __builtin_arm_stlex(42, (long long *)addr);
 // CHECK: store i64 42, i64* [[TMP:%.*]], align 8
 // CHECK: [[LOHI_ADDR:%.*]] = bitcast i64* [[TMP]] to { i32, i32 }*
 // CHECK: [[LOHI:%.*]] = load { i32, i32 }, { i32, i32 }* [[LOHI_ADDR]]
 // CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
 // CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to i64*
 // CHECK: [[TMP5:%.*]] = bitcast i64* [[TMP4]] to i8*
 // CHECK: call i32 @llvm.arm.stlexd(i32 [[LO]], i32 [[HI]], i8* [[TMP5]])
 
 // CHECK-ARM64: [[ADDR64:%.*]] = bitcast i8* %addr to i64*
-// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i64(i64 42, i64* [[ADDR64]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i64(i64 42, i64* elementtype(i64) [[ADDR64]])
 
   res |= __builtin_arm_stlex(2.71828f, (float *)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to float*
 // CHECK: [[TMP5:%.*]] = bitcast float* [[TMP4]] to i32*
 // CHECK: call i32 @llvm.arm.stlex.p0i32(i32 1076754509, i32* [[TMP5]])
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to float*
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast float* [[TMP4]] to i32*
-// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i32(i64 1076754509, i32* [[TMP5]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i32(i64 1076754509, i32* elementtype(i32) [[TMP5]])
 
   res |= __builtin_arm_stlex(3.14159, (double *)addr);
 // CHECK: store double 3.141590e+00, double* [[TMP:%.*]], align 8
 // CHECK: [[LOHI_ADDR:%.*]] = bitcast double* [[TMP]] to { i32, i32 }*
 // CHECK: [[LOHI:%.*]] = load { i32, i32 }, { i32, i32 }* [[LOHI_ADDR]]
 // CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
 // CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i8*
 // CHECK: call i32 @llvm.arm.stlexd(i32 [[LO]], i32 [[HI]], i8* [[TMP5]])
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to double*
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast double* [[TMP4]] to i64*
-// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i64(i64 4614256650576692846, i64* [[TMP5]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i64(i64 4614256650576692846, i64* elementtype(i64) [[TMP5]])
 
   res |= __builtin_arm_stlex(&var, (struct Simple **)addr);
 // CHECK: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i32*
 // CHECK: [[INTVAL:%.*]] = ptrtoint %struct.Simple* %var to i32
 // CHECK: call i32 @llvm.arm.stlex.p0i32(i32 [[INTVAL]], i32* [[TMP5]])
 
 // CHECK-ARM64: [[TMP4:%.*]] = bitcast i8* %addr to %struct.Simple**
 // CHECK-ARM64: [[TMP5:%.*]] = bitcast %struct.Simple** [[TMP4]] to i64*
 // CHECK-ARM64: [[INTVAL:%.*]] = ptrtoint %struct.Simple* %var to i64
-// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i64(i64 [[INTVAL]], i64* [[TMP5]])
+// CHECK-ARM64: call i32 @llvm.aarch64.stlxr.p0i64(i64 [[INTVAL]], i64* elementtype(i64) [[TMP5]])
 
   return res;
 }
 
 void test_clrex(void) {
 // CHECK-LABEL: @test_clrex
 // CHECK-ARM64-LABEL: @test_clrex
 

clang/test/CodeGenCXX/builtins-arm-exclusive.cpp

 // RUN: %clang_cc1 -Wall -Werror -triple thumbv8-linux-gnueabi -fno-signed-char -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -Wall -Werror -triple arm64-apple-ios7.0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARM64
 
 bool b;
 
 // CHECK-LABEL: @_Z10test_ldrexv()
 // CHECK: call i32 @llvm.arm.ldrex.p0i8(i8* @b)
 
 // CHECK-ARM64-LABEL: @_Z10test_ldrexv()
-// CHECK-ARM64: call i64 @llvm.aarch64.ldxr.p0i8(i8* @b)
+// CHECK-ARM64: call i64 @llvm.aarch64.ldxr.p0i8(i8* elementtype(i8) @b)
 
 void test_ldrex() {
   b = __builtin_arm_ldrex(&b);
 }
 
 // CHECK-LABEL: @_Z10tset_strexv()
 // CHECK: %{{.*}} = call i32 @llvm.arm.strex.p0i8(i32 1, i8* @b)
 
 // CHECK-ARM64-LABEL: @_Z10tset_strexv()
-// CHECK-ARM64: %{{.*}} = call i32 @llvm.aarch64.stxr.p0i8(i64 1, i8* @b)
+// CHECK-ARM64: %{{.*}} = call i32 @llvm.aarch64.stxr.p0i8(i64 1, i8* elementtype(i8) @b)
 
 void tset_strex() {
   __builtin_arm_strex(true, &b);
 }

llvm/lib/Bitcode/Reader/BitcodeReader.cpp

 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/Verifier.h"
for (const InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
 
       ArgNo++;
     }
   }
 
   switch (CB->getIntrinsicID()) {
   case Intrinsic::preserve_array_access_index:
   case Intrinsic::preserve_struct_access_index:
-    if (!Attrs.getParamElementType(0)) {
-      Type *ElTy = getPtrElementTypeByID(ArgTyIDs[0]);
+  case Intrinsic::aarch64_ldaxr:
+  case Intrinsic::aarch64_ldxr:
+  case Intrinsic::aarch64_stlxr:
+  case Intrinsic::aarch64_stxr: {
+    unsigned ArgNo = CB->getIntrinsicID() == Intrinsic::aarch64_stlxr ||
+                             CB->getIntrinsicID() == Intrinsic::aarch64_stxr
+                         ? 1
+                         : 0;
+    if (!Attrs.getParamElementType(ArgNo)) {
+      Type *ElTy = getPtrElementTypeByID(ArgTyIDs[ArgNo]);
       if (!ElTy)
         return error("Missing element type for elementtype upgrade");
       Attribute NewAttr = Attribute::get(Context, Attribute::ElementType, ElTy);
-      Attrs = Attrs.addParamAttribute(Context, 0, NewAttr);
+      Attrs = Attrs.addParamAttribute(Context, ArgNo, NewAttr);
     }
     break;
+  }
   default:
     break;
   }
 
   CB->setAttributes(Attrs);
   return Error::success();
 }
 

llvm/lib/IR/Verifier.cpp

 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/IntrinsicsWebAssembly.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Type.h"
case Intrinsic::experimental_vector_extract: {
     }
     break;
   }
   case Intrinsic::experimental_noalias_scope_decl: {
     NoAliasScopeDecls.push_back(cast<IntrinsicInst>(&Call));
     break;
   }
   case Intrinsic::preserve_array_access_index:
-  case Intrinsic::preserve_struct_access_index: {
+  case Intrinsic::preserve_struct_access_index:
+  case Intrinsic::aarch64_ldaxr:
+  case Intrinsic::aarch64_ldxr: {
     Type *ElemTy = Call.getParamElementType(0);
     Assert(ElemTy,
            "Intrinsic requires elementtype attribute on first argument.",
            &Call);
     break;
   }
+  case Intrinsic::aarch64_stlxr:
+  case Intrinsic::aarch64_stxr: {
+    Type *ElemTy = Call.getAttributes().getParamElementType(1);
+    Assert(ElemTy,
+           "Intrinsic requires elementtype attribute on second argument.",
+           &Call);
+    break;
+  }
   };
 }
 
 /// Carefully grab the subprogram from a local scope.
 ///
 /// This carefully grabs the subprogram from a local scope, avoiding the
 /// built-in assertions that would typically fire.
 static DISubprogram *getSubprogram(Metadata *LocalScope) {
   if (!LocalScope)

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

case Intrinsic::aarch64_neon_st4lane: {
     Info.offset = 0;
     Info.align.reset();
     // volatile stores with NEON intrinsics not supported
     Info.flags = MachineMemOperand::MOStore;
     return true;
   }
   case Intrinsic::aarch64_ldaxr:
   case Intrinsic::aarch64_ldxr: {
-    PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
+    Type *ValTy = I.getParamElementType(0);
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(PtrTy->getPointerElementType());
+    Info.memVT = MVT::getVT(ValTy);
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
-    Info.align = DL.getABITypeAlign(PtrTy->getPointerElementType());
+    Info.align = DL.getABITypeAlign(ValTy);
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
     return true;
   }
   case Intrinsic::aarch64_stlxr:
   case Intrinsic::aarch64_stxr: {
-    PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
+    Type *ValTy = I.getParamElementType(1);
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(PtrTy->getPointerElementType());
+    Info.memVT = MVT::getVT(ValTy);
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = DL.getABITypeAlign(PtrTy->getPointerElementType());
+    Info.align = DL.getABITypeAlign(ValTy);
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
     return true;
   }
   case Intrinsic::aarch64_ldaxp:
   case Intrinsic::aarch64_ldxp:
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::i128;
     Info.ptrVal = I.getArgOperand(0);
Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
 
   Type *Tys[] = { Addr->getType() };
   Intrinsic::ID Int =
       IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
   Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
 
   const DataLayout &DL = M->getDataLayout();
   IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
-  Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy);
+  CallInst *CI = Builder.CreateCall(Ldxr, Addr);
+  CI->addParamAttr(
+      0, Attribute::get(Builder.getContext(), Attribute::ElementType, ValueTy));
+  Value *Trunc = Builder.CreateTrunc(CI, IntEltTy);
 
   return Builder.CreateBitCast(Trunc, ValueTy);
 }
 
 void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
     IRBuilderBase &Builder) const {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
   Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex));
Intrinsic::ID Int =
       IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
   Type *Tys[] = { Addr->getType() };
   Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
 
   const DataLayout &DL = M->getDataLayout();
   IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
   Val = Builder.CreateBitCast(Val, IntValTy);
 
-  return Builder.CreateCall(Stxr,
-                            {Builder.CreateZExtOrBitCast(
-                                 Val, Stxr->getFunctionType()->getParamType(0)),
-                             Addr});
+  CallInst *CI = Builder.CreateCall(
+      Stxr, {Builder.CreateZExtOrBitCast(
+  CI->addParamAttr(1, Attribute::get(Builder.getContext(),
+                                     Attribute::ElementType, Val->getType()));
+  return CI;
 }
 
 bool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters(
     Type *Ty, CallingConv::ID CallConv, bool isVarArg,
     const DataLayout &DL) const {
   if (!Ty->isArrayTy()) {
     const TypeSize &TySize = Ty->getPrimitiveSizeInBits();
     return TySize.isScalable() && TySize.getKnownMinSize() > 128;

llvm/test/Bitcode/upgrade-aarch64-ldstxr.ll

+; RUN: llvm-dis < %S/upgrade-aarch64-ldstxr.bc | FileCheck %s
+
+define void @f(i32* %p) {
+; CHECK: call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32)
+  %a = call i64 @llvm.aarch64.ldxr.p0i32(i32* %p)
+; CHECK: call i32 @llvm.aarch64.stxr.p0i32(i64 0, i32* elementtype(i32)
+  %c = call i32 @llvm.aarch64.stxr.p0i32(i64 0, i32* %p)
+
+; CHECK: call i64 @llvm.aarch64.ldaxr.p0i32(i32* elementtype(i32)
+  %a2 = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %p)
+; CHECK: call i32 @llvm.aarch64.stlxr.p0i32(i64 0, i32* elementtype(i32)
+  %c2 = call i32 @llvm.aarch64.stlxr.p0i32(i64 0, i32* %p)
+  ret void
+}
+
+declare i64 @llvm.aarch64.ldxr.p0i32(i32*)
+declare i64 @llvm.aarch64.ldaxr.p0i32(i32*)
+declare i32 @llvm.aarch64.stxr.p0i32(i64, i32*)
+declare i32 @llvm.aarch64.stlxr.p0i32(i64, i32*)

llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll

define <4 x half> @test_constant_vector() {
   ret <4 x half> <half undef, half undef, half undef, half 0xH3C00>
 }
 
 define i32 @test_target_mem_intrinsic(i32* %addr) {
 ; CHECK-LABEL: name: test_target_mem_intrinsic
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[VAL:%[0-9]+]]:_(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.ldxr), [[ADDR]](p0) :: (volatile load (s32) from %ir.addr)
 ; CHECK: G_TRUNC [[VAL]](s64)
-  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32) %addr)
   %trunc = trunc i64 %val to i32
   ret i32 %trunc
 }
 
 declare i64 @llvm.aarch64.ldxr.p0i32(i32*) nounwind
 
 %zerosize_type = type {}
 

llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll

 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i8:
 ; GISEL: ldxrb w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* elementtype(i8) %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_i16
 define dso_local void @test_load_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_i16:
 ; CHECK: ldxrh w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i16:
 ; GISEL: ldxrh w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* elementtype(i16) %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_i32
 define dso_local void @test_load_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_i32:
 ; CHECK: ldxr w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i32:
 ; GISEL: ldxr w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32) %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_i64
 define dso_local void @test_load_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_i64:
 ; CHECK: ldxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_i64:
 ; GISEL: ldxr x[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) %addr)
   store i64 %val, i64* @var, align 8
   ret void
 }
 
 
 declare i64 @llvm.aarch64.ldxr.p0i8(i8*) nounwind
 declare i64 @llvm.aarch64.ldxr.p0i16(i16*) nounwind
 declare i64 @llvm.aarch64.ldxr.p0i32(i32*) nounwind
 declare i64 @llvm.aarch64.ldxr.p0i64(i64*) nounwind
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_i8
 define dso_local i32 @test_store_i8(i32, i8 %val, i8* %addr) {
 ; CHECK-LABEL: test_store_i8:
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: stxrb w0, w1, [x2]
 ; GISEL-LABEL: test_store_i8:
 ; GISEL-NOT: uxtb
 ; GISEL-NOT: and
 ; GISEL: stxrb w0, w1, [x2]
   %extval = zext i8 %val to i64
-  %res = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* %addr)
+  %res = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* elementtype(i8) %addr)
   ret i32 %res
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_i16
 define dso_local i32 @test_store_i16(i32, i16 %val, i16* %addr) {
 ; CHECK-LABEL: test_store_i16:
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: stxrh w0, w1, [x2]
 ; GISEL-LABEL: test_store_i16:
 ; GISEL-NOT: uxth
 ; GISEL-NOT: and
 ; GISEL: stxrh w0, w1, [x2]
   %extval = zext i16 %val to i64
-  %res = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* %addr)
+  %res = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* elementtype(i16) %addr)
   ret i32 %res
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_i32
 define dso_local i32 @test_store_i32(i32, i32 %val, i32* %addr) {
 ; CHECK-LABEL: test_store_i32:
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: stxr w0, w1, [x2]
 ; GISEL-LABEL: test_store_i32:
 ; GISEL-NOT: uxtw
 ; GISEL-NOT: and
 ; GISEL: stxr w0, w1, [x2]
   %extval = zext i32 %val to i64
-  %res = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* %addr)
+  %res = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* elementtype(i32) %addr)
   ret i32 %res
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_i64
 define dso_local i32 @test_store_i64(i32, i64 %val, i64* %addr) {
 ; CHECK-LABEL: test_store_i64:
 ; CHECK: stxr w0, x1, [x2]
 ; GISEL-LABEL: test_store_i64:
 ; GISEL: stxr w0, x1, [x2]
-  %res = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* %addr)
+  %res = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* elementtype(i64) %addr)
   ret i32 %res
 }
 
 declare i32 @llvm.aarch64.stxr.p0i8(i64, i8*) nounwind
 declare i32 @llvm.aarch64.stxr.p0i16(i64, i16*) nounwind
 declare i32 @llvm.aarch64.stxr.p0i32(i64, i32*) nounwind
 declare i32 @llvm.aarch64.stxr.p0i64(i64, i64*) nounwind
 

 ; CHECK: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i8:
 ; GISEL: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-DAG: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* elementtype(i8) %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i16
 define dso_local void @test_load_acquire_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_acquire_i16:
 ; CHECK: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i16:
 ; GISEL: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* elementtype(i16) %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i32
 define dso_local void @test_load_acquire_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_acquire_i32:
 ; CHECK: ldaxr w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i32:
 ; GISEL: ldaxr w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* elementtype(i32) %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i64
 define dso_local void @test_load_acquire_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_acquire_i64:
 ; CHECK: ldaxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
 ; GISEL-LABEL: test_load_acquire_i64:
 ; GISEL: ldaxr x[[LOADVAL:[0-9]+]], [x0]
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) %addr)
   store i64 %val, i64* @var, align 8
   ret void
 }
 
 
 declare i64 @llvm.aarch64.ldaxr.p0i8(i8*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i16(i16*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i32(i32*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i64(i64*) nounwind
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_release_i8
 define dso_local i32 @test_store_release_i8(i32, i8 %val, i8* %addr) {
 ; CHECK-LABEL: test_store_release_i8:
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: stlxrb w0, w1, [x2]
 ; GISEL-LABEL: test_store_release_i8:
 ; GISEL-NOT: uxtb
 ; GISEL-NOT: and
 ; GISEL: stlxrb w0, w1, [x2]
   %extval = zext i8 %val to i64
-  %res = call i32 @llvm.aarch64.stlxr.p0i8(i64 %extval, i8* %addr)
+  %res = call i32 @llvm.aarch64.stlxr.p0i8(i64 %extval, i8* elementtype(i8) %addr)
   ret i32 %res
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_release_i16
 define dso_local i32 @test_store_release_i16(i32, i16 %val, i16* %addr) {
 ; CHECK-LABEL: test_store_release_i16:
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: stlxrh w0, w1, [x2]
 ; GISEL-LABEL: test_store_release_i16:
 ; GISEL-NOT: uxth
 ; GISEL-NOT: and
 ; GISEL: stlxrh w0, w1, [x2]
   %extval = zext i16 %val to i64
-  %res = call i32 @llvm.aarch64.stlxr.p0i16(i64 %extval, i16* %addr)
+  %res = call i32 @llvm.aarch64.stlxr.p0i16(i64 %extval, i16* elementtype(i16) %addr)
   ret i32 %res
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_release_i32
 define dso_local i32 @test_store_release_i32(i32, i32 %val, i32* %addr) {
 ; CHECK-LABEL: test_store_release_i32:
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: stlxr w0, w1, [x2]
 ; GISEL-LABEL: test_store_release_i32:
 ; GISEL-NOT: uxtw
 ; GISEL-NOT: and
 ; GISEL: stlxr w0, w1, [x2]
   %extval = zext i32 %val to i64
-  %res = call i32 @llvm.aarch64.stlxr.p0i32(i64 %extval, i32* %addr)
+  %res = call i32 @llvm.aarch64.stlxr.p0i32(i64 %extval, i32* elementtype(i32) %addr)
   ret i32 %res
 }
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_release_i64
 define dso_local i32 @test_store_release_i64(i32, i64 %val, i64* %addr) {
 ; CHECK-LABEL: test_store_release_i64:
 ; CHECK: stlxr w0, x1, [x2]
 ; GISEL-LABEL: test_store_release_i64:
 ; GISEL: stlxr w0, x1, [x2]
-  %res = call i32 @llvm.aarch64.stlxr.p0i64(i64 %val, i64* %addr)
+  %res = call i32 @llvm.aarch64.stlxr.p0i64(i64 %val, i64* elementtype(i64) %addr)
   ret i32 %res
 }
 
 declare i32 @llvm.aarch64.stlxr.p0i8(i64, i8*) nounwind
 declare i32 @llvm.aarch64.stlxr.p0i16(i64, i16*) nounwind
 declare i32 @llvm.aarch64.stlxr.p0i32(i64, i32*) nounwind
 declare i32 @llvm.aarch64.stlxr.p0i64(i64, i64*) nounwind

llvm/test/CodeGen/AArch64/arm64_32-atomics.ll

 declare i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
 declare i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
 declare i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
 
 define i8 @test_ldxr_8(i8* %addr) {
 ; CHECK-LABEL: test_ldxr_8:
 ; CHECK: ldxrb w0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* elementtype(i8) %addr)
   %val8 = trunc i64 %val to i8
   ret i8 %val8
 }
 
 define i16 @test_ldxr_16(i16* %addr) {
 ; CHECK-LABEL: test_ldxr_16:
 ; CHECK: ldxrh w0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* elementtype(i16) %addr)
   %val16 = trunc i64 %val to i16
   ret i16 %val16
 }
 
 define i32 @test_ldxr_32(i32* %addr) {
 ; CHECK-LABEL: test_ldxr_32:
 ; CHECK: ldxr w0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* elementtype(i32) %addr)
   %val32 = trunc i64 %val to i32
   ret i32 %val32
 }
 
 define i64 @test_ldxr_64(i64* %addr) {
 ; CHECK-LABEL: test_ldxr_64:
 ; CHECK: ldxr x0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* elementtype(i64) %addr)
   ret i64 %val
 }
 
 declare i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
 declare i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
 declare i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
 declare i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
 
 define i8 @test_ldaxr_8(i8* %addr) {
 ; CHECK-LABEL: test_ldaxr_8:
 ; CHECK: ldaxrb w0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* elementtype(i8) %addr)
   %val8 = trunc i64 %val to i8
   ret i8 %val8
 }
 
 define i16 @test_ldaxr_16(i16* %addr) {
 ; CHECK-LABEL: test_ldaxr_16:
 ; CHECK: ldaxrh w0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* elementtype(i16) %addr)
   %val16 = trunc i64 %val to i16
   ret i16 %val16
 }
 
 define i32 @test_ldaxr_32(i32* %addr) {
 ; CHECK-LABEL: test_ldaxr_32:
 ; CHECK: ldaxr w0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* elementtype(i32) %addr)
   %val32 = trunc i64 %val to i32
   ret i32 %val32
 }
 
 define i64 @test_ldaxr_64(i64* %addr) {
 ; CHECK-LABEL: test_ldaxr_64:
 ; CHECK: ldaxr x0, [x0]
 
-  %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
+  %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) %addr)
   ret i64 %val
 }
 
 declare i32 @llvm.aarch64.stxr.p0i8(i64, i8*)
 declare i32 @llvm.aarch64.stxr.p0i16(i64, i16*)
 declare i32 @llvm.aarch64.stxr.p0i32(i64, i32*)
 declare i32 @llvm.aarch64.stxr.p0i64(i64, i64*)
 
 define i32 @test_stxr_8(i8* %addr, i8 %val) {
 ; CHECK-LABEL: test_stxr_8:
 ; CHECK: stxrb [[TMP:w[0-9]+]], w1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
   %extval = zext i8 %val to i64
-  %success = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* %addr)
+  %success = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* elementtype(i8) %addr)
   ret i32 %success
 }
 
 define i32 @test_stxr_16(i16* %addr, i16 %val) {
 ; CHECK-LABEL: test_stxr_16:
 ; CHECK: stxrh [[TMP:w[0-9]+]], w1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
   %extval = zext i16 %val to i64
-  %success = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* %addr)
+  %success = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* elementtype(i16) %addr)
   ret i32 %success
 }
 
 define i32 @test_stxr_32(i32* %addr, i32 %val) {
 ; CHECK-LABEL: test_stxr_32:
 ; CHECK: stxr [[TMP:w[0-9]+]], w1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
   %extval = zext i32 %val to i64
-  %success = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* %addr)
+  %success = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* elementtype(i32) %addr)
   ret i32 %success
 }
 
 define i32 @test_stxr_64(i64* %addr, i64 %val) {
 ; CHECK-LABEL: test_stxr_64:
 ; CHECK: stxr [[TMP:w[0-9]+]], x1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
-  %success = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* %addr)
+  %success = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* elementtype(i64) %addr)
   ret i32 %success
 }
 
 declare i32 @llvm.aarch64.stlxr.p0i8(i64, i8*)
 declare i32 @llvm.aarch64.stlxr.p0i16(i64, i16*)
 declare i32 @llvm.aarch64.stlxr.p0i32(i64, i32*)
 declare i32 @llvm.aarch64.stlxr.p0i64(i64, i64*)
 
 define i32 @test_stlxr_8(i8* %addr, i8 %val) {
 ; CHECK-LABEL: test_stlxr_8:
 ; CHECK: stlxrb [[TMP:w[0-9]+]], w1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
   %extval = zext i8 %val to i64
-  %success = call i32 @llvm.aarch64.stlxr.p0i8(i64 %extval, i8* %addr)
+  %success = call i32 @llvm.aarch64.stlxr.p0i8(i64 %extval, i8* elementtype(i8) %addr)
   ret i32 %success
 }
 
 define i32 @test_stlxr_16(i16* %addr, i16 %val) {
 ; CHECK-LABEL: test_stlxr_16:
 ; CHECK: stlxrh [[TMP:w[0-9]+]], w1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
   %extval = zext i16 %val to i64
-  %success = call i32 @llvm.aarch64.stlxr.p0i16(i64 %extval, i16* %addr)
+  %success = call i32 @llvm.aarch64.stlxr.p0i16(i64 %extval, i16* elementtype(i16) %addr)
   ret i32 %success
 }
 
 define i32 @test_stlxr_32(i32* %addr, i32 %val) {
 ; CHECK-LABEL: test_stlxr_32:
 ; CHECK: stlxr [[TMP:w[0-9]+]], w1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
   %extval = zext i32 %val to i64
-  %success = call i32 @llvm.aarch64.stlxr.p0i32(i64 %extval, i32* %addr)
+  %success = call i32 @llvm.aarch64.stlxr.p0i32(i64 %extval, i32* elementtype(i32) %addr)
   ret i32 %success
 }
 
 define i32 @test_stlxr_64(i64* %addr, i64 %val) {
 ; CHECK-LABEL: test_stlxr_64:
 ; CHECK: stlxr [[TMP:w[0-9]+]], x1, [x0]
 ; CHECK: mov w0, [[TMP]]
 
-  %success = call i32 @llvm.aarch64.stlxr.p0i64(i64 %val, i64* %addr)
+  %success = call i32 @llvm.aarch64.stlxr.p0i64(i64 %val, i64* elementtype(i64) %addr)
   ret i32 %success
 }
 
 define {i8*, i1} @test_cmpxchg_ptr(i8** %addr, i8* %cmp, i8* %new) {
 ; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel
 ; CHECK-LABEL: test_cmpxchg_ptr:
 ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
 ; CHECK:     ldaxr [[OLD:w[0-9]+]], [x0]

llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -O1 -S -mtriple=aarch64-- -atomic-expand %s | FileCheck %s
 ; RUN: opt -O1 -S -mtriple=aarch64-- -mattr=+outline-atomics -atomic-expand %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS
 
 define void @atomic_swap_f16(half* %ptr, half %val) nounwind {
 ; CHECK-LABEL: @atomic_swap_f16(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half* [[PTR:%.*]] to i16*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast half [[VAL:%.*]] to i16
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i16(i16* [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i16(i16* elementtype(i16) [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext i16 [[TMP2]] to i64
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0i16(i64 [[TMP5]], i16* [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0i16(i64 [[TMP5]], i16* elementtype(i16) [[TMP1]])
 ; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
 ; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP4]] to half
 ; CHECK-NEXT:    ret void
 ;
 ; OUTLINE-ATOMICS-LABEL: @atomic_swap_f16(
 ; OUTLINE-ATOMICS-NEXT:    [[TMP1:%.*]] = bitcast half* [[PTR:%.*]] to i16*
 ; OUTLINE-ATOMICS-NEXT:    [[TMP2:%.*]] = bitcast half [[VAL:%.*]] to i16
 ; OUTLINE-ATOMICS-NEXT:    [[TMP3:%.*]] = atomicrmw xchg i16* [[TMP1]], i16 [[TMP2]] acquire, align 2
 ; OUTLINE-ATOMICS-NEXT:    [[TMP4:%.*]] = bitcast i16 [[TMP3]] to half
 ; OUTLINE-ATOMICS-NEXT:    ret void
 ;
   %t1 = atomicrmw xchg half* %ptr, half %val acquire
   ret void
 }
 
 define void @atomic_swap_f32(float* %ptr, float %val) nounwind {
 ; CHECK-LABEL: @atomic_swap_f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[PTR:%.*]] to i32*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[VAL:%.*]] to i32
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i32(i32* [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i32(i32* elementtype(i32) [[TMP1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP2]] to i64
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0i32(i64 [[TMP5]], i32* [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0i32(i64 [[TMP5]], i32* elementtype(i32) [[TMP1]])
 ; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
 ; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret void
 ;
 ; OUTLINE-ATOMICS-LABEL: @atomic_swap_f32(
 ; OUTLINE-ATOMICS-NEXT:    [[TMP1:%.*]] = bitcast float* [[PTR:%.*]] to i32*
 ; OUTLINE-ATOMICS-NEXT:    [[TMP2:%.*]] = bitcast float [[VAL:%.*]] to i32
 ; OUTLINE-ATOMICS-NEXT:    [[TMP3:%.*]] = atomicrmw xchg i32* [[TMP1]], i32 [[TMP2]] acquire, align 4
 ; OUTLINE-ATOMICS-NEXT:    [[TMP4:%.*]] = bitcast i32 [[TMP3]] to float
 ; OUTLINE-ATOMICS-NEXT:    ret void
 ;
   %t1 = atomicrmw xchg float* %ptr, float %val acquire
   ret void
 }
 
 define void @atomic_swap_f64(double* %ptr, double %val) nounwind {
 ; CHECK-LABEL: @atomic_swap_f64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[PTR:%.*]] to i64*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double [[VAL:%.*]] to i64
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[TMP2]], i64* [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.aarch64.ldaxr.p0i64(i64* elementtype(i64) [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[TMP2]], i64* elementtype(i64) [[TMP1]])
 ; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP4]], 0
 ; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to double
 ; CHECK-NEXT:    ret void
 ;
 ; OUTLINE-ATOMICS-LABEL: @atomic_swap_f64(
 ; OUTLINE-ATOMICS-NEXT:    [[TMP1:%.*]] = bitcast double* [[PTR:%.*]] to i64*
 ; OUTLINE-ATOMICS-NEXT:    [[TMP2:%.*]] = bitcast double [[VAL:%.*]] to i64
 ; OUTLINE-ATOMICS-NEXT:    [[TMP3:%.*]] = atomicrmw xchg i64* [[TMP1]], i64 [[TMP2]] acquire, align 8
 ; OUTLINE-ATOMICS-NEXT:    [[TMP4:%.*]] = bitcast i64 [[TMP3]] to double
 ; OUTLINE-ATOMICS-NEXT:    ret void
 ;
   %t1 = atomicrmw xchg double* %ptr, double %val acquire
   ret void
 }

llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple=arm64-darwin-unknown -S -consthoist < %s | FileCheck %s
 
 ; Make sure we hoist constants out of intrinsics.
 
 define void @test_stxr(i64* %ptr) {
 ; CHECK-LABEL: @test_stxr(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CONST:%.*]] = bitcast i64 -9223372036317904832 to i64
 ; CHECK-NEXT:    [[PTR_0:%.*]] = getelementptr i64, i64* [[PTR:%.*]], i64 0
 ; CHECK-NEXT:    [[CONST_MAT:%.*]] = add i64 [[CONST]], -64
-; CHECK-NEXT:    [[BAR_0:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT]], i64* [[PTR_0]])
+; CHECK-NEXT:    [[BAR_0:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT]], i64* elementtype(i64) [[PTR_0]])
 ; CHECK-NEXT:    [[PTR_1:%.*]] = getelementptr i64, i64* [[PTR]], i64 1
-; CHECK-NEXT:    [[BAR_1:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST]], i64* [[PTR_1]])
+; CHECK-NEXT:    [[BAR_1:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST]], i64* elementtype(i64) [[PTR_1]])
 ; CHECK-NEXT:    [[PTR_2:%.*]] = getelementptr i64, i64* [[PTR]], i64 2
 ; CHECK-NEXT:    [[CONST_MAT1:%.*]] = add i64 [[CONST]], 64
-; CHECK-NEXT:    [[BAR_2:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT1]], i64* [[PTR_2]])
+; CHECK-NEXT:    [[BAR_2:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT1]], i64* elementtype(i64) [[PTR_2]])
 ; CHECK-NEXT:    [[PTR_3:%.*]] = getelementptr i64, i64* [[PTR]], i64 3
 ; CHECK-NEXT:    [[CONST_MAT2:%.*]] = add i64 [[CONST]], 128
-; CHECK-NEXT:    [[BAR_3:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT2]], i64* [[PTR_3]])
+; CHECK-NEXT:    [[BAR_3:%.*]] = call i32 @llvm.aarch64.stxr.p0i64(i64 [[CONST_MAT2]], i64* elementtype(i64) [[PTR_3]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %ptr.0 = getelementptr i64, i64* %ptr, i64 0
-  %bar.0 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904896, i64* %ptr.0)
+  %bar.0 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904896, i64* elementtype(i64) %ptr.0)
   %ptr.1 = getelementptr i64, i64* %ptr, i64 1
-  %bar.1 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904832,  i64* %ptr.1)
+  %bar.1 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904832,  i64* elementtype(i64) %ptr.1)
   %ptr.2 = getelementptr i64, i64* %ptr, i64 2
-  %bar.2 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904768, i64* %ptr.2)
+  %bar.2 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904768, i64* elementtype(i64) %ptr.2)
   %ptr.3 = getelementptr i64, i64* %ptr, i64 3
-  %bar.3 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904704, i64* %ptr.3)
+  %bar.3 = call i32 @llvm.aarch64.stxr.p0i64(i64 -9223372036317904704, i64* elementtype(i64) %ptr.3)
   ret void
 }
 
 declare i32 @llvm.aarch64.stxr.p0i64(i64 , i64*)
 
 define i64 @test_udiv(i64 %x) {
 ; CHECK-LABEL: @test_udiv(
 ; CHECK-NEXT:  entry:

llvm/test/Verifier/aarch64-ldstxr.ll

+; RUN: not opt -passes=verify -S < %s 2>&1 | FileCheck %s
+
+define void @f(i32* %p) {
+; CHECK: Intrinsic requires elementtype attribute on first argument
+  %a = call i64 @llvm.aarch64.ldxr.p0i32(i32* %p)
+; CHECK: Intrinsic requires elementtype attribute on second argument
+  %c = call i32 @llvm.aarch64.stxr.p0i32(i64 0, i32* %p)
+
+; CHECK: Intrinsic requires elementtype attribute on first argument
+  %a2 = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %p)
+; CHECK: Intrinsic requires elementtype attribute on second argument
+  %c2 = call i32 @llvm.aarch64.stlxr.p0i32(i64 0, i32* %p)
+  ret void
+}
+
+declare i64 @llvm.aarch64.ldxr.p0i32(i32*)
+declare i64 @llvm.aarch64.ldaxr.p0i32(i32*)
+declare i32 @llvm.aarch64.stxr.p0i32(i64, i32*)
+declare i32 @llvm.aarch64.stlxr.p0i32(i64, i32*)