diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1018,16 +1018,19 @@ SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo); + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo); + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, - MachinePointerInfo DstPtrInfo); + MachinePointerInfo DstPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, unsigned DstAlign, SDValue Src, unsigned SrcAlign, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6256,7 +6256,8 @@ uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Turn a memcpy of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -6357,7 +6358,7 @@ Store = DAG.getStore( Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, AAInfo); OutChains.push_back(Store); } } @@ -6381,13 +6382,13 @@ ISD::EXTLOAD, dl, NVT, Chain, DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), SrcPtrInfo.getWithOffset(SrcOff), VT, - commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags); + commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, AAInfo); OutLoadChains.push_back(Value.getValue(1)); Store = DAG.getTruncStore( Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags); + DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, AAInfo); OutStoreChains.push_back(Store); } SrcOff += VTSize; @@ -6446,7 +6447,8 @@ uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Turn a memmove of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -6507,10 +6509,10 @@ if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; - Value = - DAG.getLoad(VT, dl, Chain, - DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), - SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags); + Value = DAG.getLoad( + VT, dl, Chain, + DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), + SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, AAInfo); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); SrcOff += VTSize; @@ -6522,10 +6524,10 @@ unsigned VTSize = VT.getSizeInBits() / 8; SDValue Store; - Store = - DAG.getStore(Chain, dl, LoadValues[i], - DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); + Store = DAG.getStore( + Chain, dl, LoadValues[i], + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, AAInfo); OutChains.push_back(Store); DstOff += VTSize; } @@ -6554,7 +6556,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, - MachinePointerInfo DstPtrInfo) { + MachinePointerInfo DstPtrInfo, + const AAMDNodes &AAInfo) { // Turn a memset of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -6626,7 +6629,8 @@ Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), DstPtrInfo.getWithOffset(DstOff), Alignment, - isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone); + isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, + AAInfo); OutChains.push_back(Store); DstOff += VT.getSizeInBits() / 8; Size -= VTSize; @@ -6649,7 +6653,8 @@ SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Check to see if we should lower the memcpy to loads and stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -6660,7 +6665,7 @@ SDValue Result = getMemcpyLoadsAndStores( *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, false, DstPtrInfo, SrcPtrInfo); + isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); if (Result.getNode()) return Result; } @@ -6681,7 +6686,7 @@ assert(ConstantSize && "AlwaysInline requires a constant size!"); return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, true, DstPtrInfo, SrcPtrInfo); + isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo); } checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); @@ -6763,7 +6768,8 @@ SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Check to see if we should lower the memmove to loads and stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -6774,7 +6780,7 @@ SDValue Result = getMemmoveLoadsAndStores( *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, false, DstPtrInfo, SrcPtrInfo); + isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); if (Result.getNode()) return Result; } @@ -6864,7 +6870,8 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, - MachinePointerInfo DstPtrInfo) { + MachinePointerInfo DstPtrInfo, + const AAMDNodes &AAInfo) { // Check to see if we should lower the memset to stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -6875,7 +6882,7 @@ SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, DstPtrInfo); + isVol, DstPtrInfo, AAInfo); if (Result.getNode()) return Result; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5744,6 +5744,16 @@ llvm_unreachable("expected corresponding call to preallocated setup/arg"); } +// Prepare AAInfo for memory transfer intrinsics. Simply copy the scoped AA +// metadata only. +static inline AAMDNodes prepareAAInfoForMemTransferInst(const CallInst &I) { + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + // Clear TBAA metadata. + AAInfo.TBAA = AAInfo.TBAAStruct = nullptr; + return AAInfo; +} + /// Lower the call to the specified intrinsic function. void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { @@ -5826,10 +5836,11 @@ // FIXME: Support passing different dest/src alignments to the memcpy DAG // node. SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo = prepareAAInfoForMemTransferInst(I); SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); updateDAGForMaybeTailCall(MC); return; } @@ -5847,10 +5858,11 @@ bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); // FIXME: Support passing different dest/src alignments to the memcpy DAG // node. + AAMDNodes AAInfo = prepareAAInfoForMemTransferInst(I); SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol, /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); updateDAGForMaybeTailCall(MC); return; } @@ -5864,8 +5876,9 @@ bool isVol = MSI.isVolatile(); bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo = prepareAAInfoForMemTransferInst(I); SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC, - MachinePointerInfo(I.getArgOperand(0))); + MachinePointerInfo(I.getArgOperand(0)), AAInfo); updateDAGForMaybeTailCall(MS); return; } @@ -5883,9 +5896,10 @@ // FIXME: Support passing different dest/src alignments to the memmove DAG // node. SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo = prepareAAInfoForMemTransferInst(I); SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); updateDAGForMaybeTailCall(MM); return; } @@ -7694,10 +7708,11 @@ // because the return pointer needs to be adjusted by the size of // the copied memory. SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo = prepareAAInfoForMemTransferInst(I); SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false, /*isTailCall=*/false, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); assert(MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **"); DAG.setRoot(MC); diff --git a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll @@ -0,0 +1,130 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -o - %s | FileCheck %s +; RUN: llc -mtriple=aarch64-linux-gnu -stop-after=finalize-isel -o - %s | FileCheck --check-prefix=MIR %s + +; MIR-DAG: ![[DOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"bax"} +; MIR-DAG: ![[SCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"bax: %p"} +; MIR-DAG: ![[SCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"bax: %q"} +; MIR-DAG: ![[SET0:[0-9]+]] = !{![[SCOPE0]]} +; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} + +; MIR-LABEL: name: test_memcpy +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memcpy: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: ldr [[PVAL:q[0-9]+]], [x0, #16] +; CHECK-DAG: add w8, [[Q0]], [[Q1]] +; CHECK: str [[PVAL]], [x0] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memcpy_inline +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memcpy_inline(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memcpy_inline: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: ldr [[PVAL:q[0-9]+]], [x0, #16] +; CHECK-DAG: add w8, [[Q0]], [[Q1]] +; CHECK: str [[PVAL]], [x0] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memmove +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memmove(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memmove: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: ldr [[PVAL:q[0-9]+]], [x0, #16] +; CHECK-DAG: add w8, [[Q0]], [[Q1]] +; CHECK: str [[PVAL]], [x0] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memmove.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memset +; MIR: %2:gpr64 = MOVi64imm -6148914691236517206 +; MIR-NEXT: STRXui %2, %0, 1 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: STRXui %2, %0, 0 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memset(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memset: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: mov [[PVAL:x[0-9]+]], #-6148914691236517206 +; CHECK: stp [[PVAL]], [[PVAL]], [x0] +; CHECK: add w8, [[Q0]], [[Q1]] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + tail call void @llvm.memset.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8 170, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_mempcpy +; MIR: %2:fpr128 = LDRQui %0, 1 :: (load 16 from %ir.p1, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: STRQui killed %2, %0, 0 :: (store 16 into %ir.p0, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_mempcpy(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_mempcpy: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: ldr [[PVAL:q[0-9]+]], [x0, #16] +; CHECK-DAG: add w8, [[Q0]], [[Q1]] +; CHECK: str [[PVAL]], [x0] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + %call = tail call i8* @mempcpy(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1 immarg) +declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) + +declare i8* @mempcpy(i8*, i8*, i64) + +!0 = distinct !{!0, !"bax"} +!1 = distinct !{!1, !0, !"bax: %p"} +!2 = !{!1} +!3 = distinct !{!3, !0, !"bax: %q"} +!4 = !{!3} diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll @@ -0,0 +1,108 @@ +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -o - %s | FileCheck %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -stop-after=finalize-isel -o - %s | FileCheck --check-prefix=MIR %s + +; Ensure that the scoped AA is attached on loads/stores lowered from mem ops. + +; Re-evaluate the slot numbers of scopes as that numbering could be changed run-by-run. + +; MIR-DAG: ![[DOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"bax"} +; MIR-DAG: ![[SCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"bax: %p"} +; MIR-DAG: ![[SCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"bax: %q"} +; MIR-DAG: ![[SET0:[0-9]+]] = !{![[SCOPE0]]} +; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} + +; MIR-LABEL: name: test_memcpy +; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +define i32 @test_memcpy(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; Check loads of %q are scheduled ahead of that store of the memcpy on %p. +; CHECK-LABEL: test_memcpy: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: global_load_dwordx4 [[PVAL:v\[[0-9]+:[0-9]+\]]], v[0:1], off offset:16 +; CHECK-DAG: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: global_store_dwordx4 v[0:1], [[PVAL]], off +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4 + %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)* + tail call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memcpy_inline +; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +define i32 @test_memcpy_inline(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; Check loads of %q are scheduled ahead of that store of the memcpy on %p. +; CHECK-LABEL: test_memcpy_inline: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: global_load_dwordx4 [[PVAL:v\[[0-9]+:[0-9]+\]]], v[0:1], off offset:16 +; CHECK-DAG: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: global_store_dwordx4 v[0:1], [[PVAL]], off +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4 + %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)* + tail call void @llvm.memcpy.inline.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memmove +; MIR: %8:vreg_128 = GLOBAL_LOAD_DWORDX4 %9, 16, 0, implicit $exec :: (load 16 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +; MIR: GLOBAL_STORE_DWORDX4 %10, killed %8, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +define i32 @test_memmove(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; Check loads of %q are scheduled ahead of that store of the memmove on %p. +; CHECK-LABEL: test_memmove: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: global_load_dwordx4 [[PVAL:v\[[0-9]+:[0-9]+\]]], v[0:1], off offset:16 +; CHECK-DAG: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: global_store_dwordx4 v[0:1], [[PVAL]] +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4 + %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)* + tail call void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memset +; MIR: GLOBAL_STORE_DWORDX4 killed %10, killed %11, 0, 0, implicit $exec :: (store 16 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]], addrspace 1) +define i32 @test_memset(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; Check loads of %q are scheduled ahead of that store of the memset on %p. +; CHECK-LABEL: test_memset: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: v_mov_b32_e32 v[[PVAL:[0-9]+]], 0xaaaaaaaa +; CHECK: global_store_dwordx4 v[0:1], v{{\[}}[[PVAL]]{{:[0-9]+\]}}, off +; CHECK: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + tail call void @llvm.memset.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 170, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* noalias nocapture writeonly, i8 addrspace(1)* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memcpy.inline.p1i8.p1i8.i64(i8 addrspace(1)* noalias nocapture writeonly, i8 addrspace(1)* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* nocapture writeonly, i8 addrspace(1)* nocapture readonly, i64, i1 immarg) +declare void @llvm.memset.p1i8.i64(i8 addrspace(1)* nocapture writeonly, i8, i64, i1 immarg) + +!0 = distinct !{!0, !"bax"} +!1 = distinct !{!1, !0, !"bax: %p"} +!2 = !{!1} +!3 = distinct !{!3, !0, !"bax: %q"} +!4 = !{!3} diff --git a/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll b/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/memcpy-scoped-aa.ll @@ -0,0 +1,106 @@ +; RUN: llc -mtriple=x86_64-linux-gnu -stop-after=finalize-isel -o - %s | FileCheck --check-prefix=MIR %s + +; Ensure that the scoped AA is attached on loads/stores lowered from mem ops. + +; Re-evaluate the slot numbers of scopes as that numbering could be changed run-by-run. + +; MIR-DAG: ![[DOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"bax"} +; MIR-DAG: ![[SCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"bax: %p"} +; MIR-DAG: ![[SCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"bax: %q"} +; MIR-DAG: ![[SET0:[0-9]+]] = !{![[SCOPE0]]} +; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} + +; MIR-LABEL: name: test_memcpy +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) { + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memcpy_inline +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memcpy_inline(i32* nocapture %p, i32* nocapture readonly %q) { + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memmove +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memmove(i32* nocapture %p, i32* nocapture readonly %q) { + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memmove.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_memset +; MIR: %2:gr64 = MOV64ri -6148914691236517206 +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, %2 :: (store 8 into %ir.p0 + 8, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, %2 :: (store 8 into %ir.p0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_memset(i32* nocapture %p, i32* nocapture readonly %q) { + %p0 = bitcast i32* %p to i8* + tail call void @llvm.memset.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8 170, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; MIR-LABEL: name: test_mempcpy +; MIR: %2:gr64 = MOV64rm %0, 1, $noreg, 16, $noreg :: (load 8 from %ir.p1, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: %3:gr64 = MOV64rm %0, 1, $noreg, 24, $noreg :: (load 8 from %ir.p1 + 8, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 8, $noreg, killed %3 :: (store 8 into %ir.p0 + 8, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +; MIR-NEXT: MOV64mr %0, 1, $noreg, 0, $noreg, killed %2 :: (store 8 into %ir.p0, align 1, !alias.scope ![[SET0]], !noalias ![[SET1]]) +define i32 @test_mempcpy(i32* nocapture %p, i32* nocapture readonly %q) { + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + %call = tail call i8* @mempcpy(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1 immarg) +declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) + +declare i8* @mempcpy(i8*, i8*, i64) + +!0 = distinct !{!0, !"bax"} +!1 = distinct !{!1, !0, !"bax: %p"} +!2 = !{!1} +!3 = distinct !{!3, !0, !"bax: %q"} +!4 = !{!3}