diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -707,6 +707,10 @@ Result.NoAlias = NoAlias; return Result; } + + /// Create a new AAMDNode as the result after merging two pointers from two + /// mem ops into a single one. + AAMDNodes merge(const AAMDNodes &Other) const; }; // Specialize DenseMapInfo for AAMDNodes. @@ -1490,6 +1494,15 @@ // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_ISA_CONVERSION_FUNCTIONS(NamedMDNode, LLVMNamedMDNodeRef) +inline AAMDNodes AAMDNodes::merge(const AAMDNodes &Other) const { + AAMDNodes Result; + Result.TBAA = MDNode::getMostGenericTBAA(TBAA, Other.TBAA); + Result.TBAAStruct = nullptr; + Result.Scope = MDNode::getMostGenericAliasScope(Scope, Other.Scope); + Result.NoAlias = MDNode::intersect(NoAlias, Other.NoAlias); + return Result; +} + } // end namespace llvm #endif // LLVM_IR_METADATA_H diff --git a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp --- a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp @@ -522,20 +522,14 @@ } void Instruction::getAAMetadata(AAMDNodes &N, bool Merge) const { - if (Merge) { - N.TBAA = - MDNode::getMostGenericTBAA(N.TBAA, getMetadata(LLVMContext::MD_tbaa)); - N.TBAAStruct = nullptr; - N.Scope = MDNode::getMostGenericAliasScope( - N.Scope, getMetadata(LLVMContext::MD_alias_scope)); - N.NoAlias = - MDNode::intersect(N.NoAlias, getMetadata(LLVMContext::MD_noalias)); - } else { - N.TBAA = getMetadata(LLVMContext::MD_tbaa); - N.TBAAStruct = getMetadata(LLVMContext::MD_tbaa_struct); - N.Scope = getMetadata(LLVMContext::MD_alias_scope); - N.NoAlias = getMetadata(LLVMContext::MD_noalias); - } + AAMDNodes AAInfo; + AAInfo.TBAA = getMetadata(LLVMContext::MD_tbaa); + AAInfo.TBAAStruct = getMetadata(LLVMContext::MD_tbaa_struct); + AAInfo.Scope = getMetadata(LLVMContext::MD_alias_scope); + AAInfo.NoAlias = getMetadata(LLVMContext::MD_noalias); + if (Merge) + AAInfo = AAInfo.merge(N); + N = AAInfo; } static const MDNode *createAccessTag(const MDNode *AccessType) { diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -16876,6 +16876,22 @@ unsigned SizeInBits = NumStores * ElementSizeBits; unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1; + Optional Flags; + AAMDNodes AAInfo; + for (unsigned I = 0; I != NumStores; ++I) { + StoreSDNode *St = cast(StoreNodes[I].MemNode); + if (!Flags) { + Flags = St->getMemOperand()->getFlags(); + AAInfo = St->getAAInfo(); + continue; + } + // Skip merging if there's an inconsistent flag. + if (Flags != St->getMemOperand()->getFlags()) + return false; + // Merge AA metadata. + AAInfo = AAInfo.merge(St->getAAInfo()); + } + EVT StoreTy; if (UseVector) { unsigned Elts = NumStores * NumMemElts; @@ -16993,9 +17009,9 @@ // make sure we use trunc store if it's necessary to be legal. SDValue NewStore; if (!UseTrunc) { - NewStore = - DAG.getStore(NewChain, DL, StoredVal, FirstInChain->getBasePtr(), - FirstInChain->getPointerInfo(), FirstInChain->getAlign()); + NewStore = DAG.getStore(NewChain, DL, StoredVal, FirstInChain->getBasePtr(), + FirstInChain->getPointerInfo(), + FirstInChain->getAlign(), Flags.getValue(), AAInfo); } else { // Must be realized as a trunc store EVT LegalizedStoredValTy = TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType()); @@ -17007,7 +17023,7 @@ NewStore = DAG.getTruncStore( NewChain, DL, ExtendedStoreVal, FirstInChain->getBasePtr(), FirstInChain->getPointerInfo(), StoredVal.getValueType() /*TVT*/, - FirstInChain->getAlign(), FirstInChain->getMemOperand()->getFlags()); + FirstInChain->getAlign(), Flags.getValue(), AAInfo); } // Replace all merged stores with the new store. diff --git a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll @@ -0,0 +1,56 @@ +; RUN: llc %s -mtriple=aarch64-eabi -stop-after=finalize-isel -o - | FileCheck --check-prefix=MIR %s + +; Ensure the scoped AA metadata is still retained after store merging. + +; MIR-DAG: ![[DOMAIN:[0-9]+]] = distinct !{!{{[0-9]+}}, !"domain"} +; MIR-DAG: ![[SCOPE0:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"scope 0"} +; MIR-DAG: ![[SCOPE1:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"scope 1"} +; MIR-DAG: ![[SCOPE2:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"scope 2"} +; MIR-DAG: ![[SCOPE3:[0-9]+]] = distinct !{!{{[0-9]+}}, ![[DOMAIN]], !"scope 3"} +; MIR-DAG: ![[SET0:[0-9]+]] = !{![[SCOPE0]]} +; MIR-DAG: ![[SET1:[0-9]+]] = !{![[SCOPE1]]} +; MIR-DAG: ![[SET2:[0-9]+]] = !{![[SCOPE2]]} +; MIR-DAG: ![[SET3:[0-9]+]] = !{![[SCOPE3]]} + +define void @blam(<3 x float>* %g0, <3 x float>* %g1) { +; MIR-LABEL: name: blam +; MIR: STRDui killed %5, %1, 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope ![[SET1]], !noalias ![[SET0]]) + %tmp4 = getelementptr inbounds <3 x float>, <3 x float>* %g1, i64 0, i64 0 + %tmp5 = load <3 x float>, <3 x float>* %g0, align 4, !alias.scope !0, !noalias !1 + %tmp6 = extractelement <3 x float> %tmp5, i64 0 + store float %tmp6, float* %tmp4, align 4, !alias.scope !1, !noalias !0 + %tmp7 = getelementptr inbounds float, float* %tmp4, i64 1 + %tmp8 = load <3 x float>, <3 x float>* %g0, align 4, !alias.scope !0, !noalias !1 + %tmp9 = extractelement <3 x float> %tmp8, i64 1 + store float %tmp9, float* %tmp7, align 4, !alias.scope !1, !noalias !0 + ret void; +} + +; Ensure new scoped AA metadata are calculated after merging stores. +define void @blam1(<3 x float>* %g0, <3 x float>* %g1) { +; MIR-LABEL: name: blam1 +; MIR: machineMetadataNodes: +; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE2]], ![[SCOPE1]]} +; MIR-DAG: ![[MMSET1:[0-9]+]] = !{} +; MIR: body: +; MIR: STRDui killed %5, %1, 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET1]]) + %tmp4 = getelementptr inbounds <3 x float>, <3 x float>* %g1, i64 0, i64 0 + %tmp5 = load <3 x float>, <3 x float>* %g0, align 4, !alias.scope !0, !noalias !1 + %tmp6 = extractelement <3 x float> %tmp5, i64 0 + store float %tmp6, float* %tmp4, align 4, !alias.scope !1, !noalias !0 + %tmp7 = getelementptr inbounds float, float* %tmp4, i64 1 + %tmp8 = load <3 x float>, <3 x float>* %g0, align 4, !alias.scope !0, !noalias !1 + %tmp9 = extractelement <3 x float> %tmp8, i64 1 + store float %tmp9, float* %tmp7, align 4, !alias.scope !5, !noalias !6 + ret void; +} + +!0 = !{!2} +!1 = !{!3} +!2 = !{!2, !4, !"scope 0"} +!3 = !{!3, !4, !"scope 1"} +!4 = !{!4, !"domain"} +!5 = !{!7} +!6 = !{!8} +!7 = !{!7, !4, !"scope 2"} +!8 = !{!8, !4, !"scope 3"}