diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1018,16 +1018,19 @@ SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo); + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo); + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, - MachinePointerInfo DstPtrInfo); + MachinePointerInfo DstPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, unsigned DstAlign, SDValue Src, unsigned SrcAlign, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6318,7 +6318,8 @@ uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Turn a memcpy of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -6419,7 +6420,7 @@ Store = DAG.getStore( Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, AAInfo); OutChains.push_back(Store); } } @@ -6443,13 +6444,13 @@ ISD::EXTLOAD, dl, NVT, Chain, DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), SrcPtrInfo.getWithOffset(SrcOff), VT, - commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags); + commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, AAInfo); OutLoadChains.push_back(Value.getValue(1)); Store = DAG.getTruncStore( Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags); + DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, AAInfo); OutStoreChains.push_back(Store); } SrcOff += VTSize; @@ -6508,7 +6509,8 @@ uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Turn a memmove of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -6569,10 +6571,10 @@ if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; - Value = - DAG.getLoad(VT, dl, Chain, - DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), - SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags); + Value = DAG.getLoad( + VT, dl, Chain, + DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), + SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, AAInfo); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); SrcOff += VTSize; @@ -6584,10 +6586,10 @@ unsigned VTSize = VT.getSizeInBits() / 8; SDValue Store; - Store = - DAG.getStore(Chain, dl, LoadValues[i], - DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); + Store = DAG.getStore( + Chain, dl, LoadValues[i], + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, AAInfo); OutChains.push_back(Store); DstOff += VTSize; } @@ -6616,7 +6618,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, - MachinePointerInfo DstPtrInfo) { + MachinePointerInfo DstPtrInfo, + const AAMDNodes &AAInfo) { // Turn a memset of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -6688,7 +6691,8 @@ Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), DstPtrInfo.getWithOffset(DstOff), Alignment, - isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone); + isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, + AAInfo); OutChains.push_back(Store); DstOff += VT.getSizeInBits() / 8; Size -= VTSize; @@ -6711,7 +6715,8 @@ SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Check to see if we should lower the memcpy to loads and stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -6722,7 +6727,7 @@ SDValue Result = getMemcpyLoadsAndStores( *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, false, DstPtrInfo, SrcPtrInfo); + isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); if (Result.getNode()) return Result; } @@ -6743,7 +6748,7 @@ assert(ConstantSize && "AlwaysInline requires a constant size!"); return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, true, DstPtrInfo, SrcPtrInfo); + isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo); } checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); @@ -6825,7 +6830,8 @@ SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo) { + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo) { // Check to see if we should lower the memmove to loads and stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -6836,7 +6842,7 @@ SDValue Result = getMemmoveLoadsAndStores( *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, false, DstPtrInfo, SrcPtrInfo); + isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); if (Result.getNode()) return Result; } @@ -6926,7 +6932,8 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, - MachinePointerInfo DstPtrInfo) { + MachinePointerInfo DstPtrInfo, + const AAMDNodes &AAInfo) { // Check to see if we should lower the memset to stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -6937,7 +6944,7 @@ SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, DstPtrInfo); + isVol, DstPtrInfo, AAInfo); if (Result.getNode()) return Result; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5807,10 +5807,14 @@ // FIXME: Support passing different dest/src alignments to the memcpy DAG // node. SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + // Clear TBAA metadata. + AAInfo.TBAA = AAInfo.TBAAStruct = nullptr; SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); updateDAGForMaybeTailCall(MC); return; } @@ -5828,10 +5832,14 @@ bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); // FIXME: Support passing different dest/src alignments to the memcpy DAG // node. + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + // Clear TBAA metadata. + AAInfo.TBAA = AAInfo.TBAAStruct = nullptr; SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol, /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); updateDAGForMaybeTailCall(MC); return; } @@ -5845,8 +5853,12 @@ bool isVol = MSI.isVolatile(); bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + // Clear TBAA metadata. + AAInfo.TBAA = AAInfo.TBAAStruct = nullptr; SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC, - MachinePointerInfo(I.getArgOperand(0))); + MachinePointerInfo(I.getArgOperand(0)), AAInfo); updateDAGForMaybeTailCall(MS); return; } @@ -5864,9 +5876,13 @@ // FIXME: Support passing different dest/src alignments to the memmove DAG // node. SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + // Clear TBAA metadata. + AAInfo.TBAA = AAInfo.TBAAStruct = nullptr; SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); updateDAGForMaybeTailCall(MM); return; } @@ -7649,10 +7665,14 @@ // because the return pointer needs to be adjusted by the size of // the copied memory. SDValue Root = isVol ? getRoot() : getMemoryRoot(); + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + // Clear TBAA metadata. + AAInfo.TBAA = AAInfo.TBAAStruct = nullptr; SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false, /*isTailCall=*/false, MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1))); + MachinePointerInfo(I.getArgOperand(1)), AAInfo); assert(MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **"); DAG.setRoot(MC); diff --git a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll @@ -0,0 +1,66 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -o - %s | FileCheck %s + +define i32 @test_memcpy(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memcpy: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: ldr [[PVAL:q[0-9]+]], [x0, #16] +; CHECK-DAG: add w8, [[Q0]], [[Q1]] +; CHECK: str [[PVAL]], [x0] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +define i32 @test_memmove(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memmove: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: ldr [[PVAL:q[0-9]+]], [x0, #16] +; CHECK-DAG: add w8, [[Q0]], [[Q1]] +; CHECK: str [[PVAL]], [x0] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + %add.ptr = getelementptr inbounds i32, i32* %p, i64 4 + %p1 = bitcast i32* %add.ptr to i8* + tail call void @llvm.memmove.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +define i32 @test_memset(i32* nocapture %p, i32* nocapture readonly %q) { +; CHECK-LABEL: test_memset: +; CHECK-DAG: ldp [[Q0:w[0-9]+]], [[Q1:w[0-9]+]], [x1] +; CHECK-DAG: mov [[PVAL:x[0-9]+]], #-6148914691236517206 +; CHECK: stp [[PVAL]], [[PVAL]], [x0] +; CHECK: add w8, [[Q0]], [[Q1]] +; CHECK: mov w0, w8 +; CHECK: ret + %p0 = bitcast i32* %p to i8* + tail call void @llvm.memset.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) %p0, i8 170, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32* %q, i64 1 + %v1 = load i32, i32* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1 immarg) +declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) + +!0 = distinct !{!0, !"bax"} +!1 = distinct !{!1, !0, !"bax: %p"} +!2 = !{!1} +!3 = distinct !{!3, !0, !"bax: %q"} +!4 = !{!3} diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/memcpy-scoped-aa.ll @@ -0,0 +1,66 @@ +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -o - %s | FileCheck %s + +; Check loads of %q are scheduled ahead of that store of the memcpy on %p. +define i32 @test_memcpy(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; CHECK-LABEL: test_memcpy: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: global_load_dwordx4 [[PVAL:v\[[0-9]+:[0-9]+\]]], v[0:1], off offset:16 +; CHECK-DAG: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: global_store_dwordx4 v[0:1], [[PVAL]], off +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4 + %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)* + tail call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; Check loads of %q are scheduled ahead of that store of the memmove on %p. +define i32 @test_memmove(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; CHECK-LABEL: test_memmove: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: global_load_dwordx4 [[PVAL:v\[[0-9]+:[0-9]+\]]], v[0:1], off offset:16 +; CHECK-DAG: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: global_store_dwordx4 v[0:1], [[PVAL]] +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + %add.ptr = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 4 + %p1 = bitcast i32 addrspace(1)* %add.ptr to i8 addrspace(1)* + tail call void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p1, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +; Check loads of %q are scheduled ahead of that store of the memset on %p. +define i32 @test_memset(i32 addrspace(1)* nocapture %p, i32 addrspace(1)* nocapture readonly %q) { +; CHECK-LABEL: test_memset: +; CHECK-DAG: global_load_dwordx2 v{{\[}}[[Q0:[0-9]+]]:[[Q1:[0-9]+]]{{\]}}, v[2:3], off +; CHECK-DAG: v_mov_b32_e32 v[[PVAL:[0-9]+]], 0xaaaaaaaa +; CHECK: global_store_dwordx4 v[0:1], v{{\[}}[[PVAL]]{{:[0-9]+\]}}, off +; CHECK: v_add_nc_u32_e32 v{{[0-9]+}}, v[[Q0]], v[[Q1]] +; CHECK: s_setpc_b64 s[30:31] + %p0 = bitcast i32 addrspace(1)* %p to i8 addrspace(1)* + tail call void @llvm.memset.p1i8.i64(i8 addrspace(1)* noundef nonnull align 4 dereferenceable(16) %p0, i8 170, i64 16, i1 false), !alias.scope !2, !noalias !4 + %v0 = load i32, i32 addrspace(1)* %q, align 4, !alias.scope !4, !noalias !2 + %q1 = getelementptr inbounds i32, i32 addrspace(1)* %q, i64 1 + %v1 = load i32, i32 addrspace(1)* %q1, align 4, !alias.scope !4, !noalias !2 + %add = add i32 %v0, %v1 + ret i32 %add +} + +declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* noalias nocapture writeonly, i8 addrspace(1)* noalias nocapture readonly, i64, i1 immarg) +declare void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* nocapture writeonly, i8 addrspace(1)* nocapture readonly, i64, i1 immarg) +declare void @llvm.memset.p1i8.i64(i8 addrspace(1)* nocapture writeonly, i8, i64, i1 immarg) + +!0 = distinct !{!0, !"bax"} +!1 = distinct !{!1, !0, !"bax: %p"} +!2 = !{!1} +!3 = distinct !{!3, !0, !"bax: %q"} +!4 = !{!3}