diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -2724,6 +2724,10 @@ if (DoesKMove) K->setMetadata(Kind, JMD); break; + case LLVMContext::MD_nontemporal: + // Preserve !nontemporal if it is present on both instructions. + K->setMetadata(Kind, JMD); + break; } } // Set !invariant.group from J if J has it. If both instructions have it @@ -2746,7 +2750,8 @@ LLVMContext::MD_invariant_group, LLVMContext::MD_align, LLVMContext::MD_dereferenceable, LLVMContext::MD_dereferenceable_or_null, - LLVMContext::MD_access_group, LLVMContext::MD_preserve_access_index}; + LLVMContext::MD_access_group, LLVMContext::MD_preserve_access_index, + LLVMContext::MD_nontemporal}; combineMetadata(K, J, KnownIDs, KDominatesJ); } @@ -2831,7 +2836,7 @@ LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group, LLVMContext::MD_nonnull, LLVMContext::MD_access_group, LLVMContext::MD_preserve_access_index, - LLVMContext::MD_noundef}; + LLVMContext::MD_noundef, LLVMContext::MD_nontemporal}; combineMetadata(ReplInst, I, KnownIDs, false); } diff --git a/llvm/test/Transforms/GVN/nontemporal.ll b/llvm/test/Transforms/GVN/nontemporal.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/GVN/nontemporal.ll @@ -0,0 +1,30 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -passes=gvn -S < %s | FileCheck %s + +; Check that !nontemporal metadata is preserved only if both original +; instructions are marked. +define i64 @one_nontemporal(ptr %p) { +; CHECK-LABEL: @one_nontemporal( +; CHECK-NEXT: [[A:%.*]] = load i64, ptr [[P:%.*]], align 4 +; CHECK-NEXT: [[C:%.*]] = add i64 [[A]], [[A]] +; CHECK-NEXT: ret i64 [[C]] +; + %a = load i64, ptr %p + %b = load i64, ptr %p, !nontemporal !0 + %c = add i64 %a, %b + ret i64 %c +} + +define i64 @both_nontemporal(ptr %p) { +; CHECK-LABEL: @both_nontemporal( +; CHECK-NEXT: [[A:%.*]] = load i64, ptr [[P:%.*]], align 4, !nontemporal !0 +; CHECK-NEXT: [[C:%.*]] = add i64 [[A]], [[A]] +; CHECK-NEXT: ret i64 [[C]] +; + %a = load i64, ptr %p, !nontemporal !0 + %b = load i64, ptr %p, !nontemporal !0 + %c = add i64 %a, %b + ret i64 %c +} + +!0 = !{i32 1} diff --git a/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll b/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll --- a/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll +++ b/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll @@ -51,5 +51,49 @@ ret ptr %c } +; Check that nontemporal markings are propagated if both original stores are +; marked. +; CHECK-LABEL: @nontemporal( +; CHECK-LABEL: if.end: +; CHECK: !nontemporal +; CHECK: ret void +define void @nontemporal(i1 zeroext %flag, ptr %p) { +entry: + br i1 %flag, label %if.then, label %if.else + +if.then: + %a = load ptr, ptr %p + store ptr %p, ptr %a, align 8, !nontemporal !1 + br label %if.end + +if.else: + %b = load ptr, ptr %p + store ptr %p, ptr %b, align 8, !nontemporal !1 + br label %if.end + +if.end: + ret void +} + +; CHECK-LABEL: @nontemporal_mismatch( +; CHECK-NOT: !nontemporal +define void @nontemporal_mismatch(i1 zeroext %flag, ptr %p) { +entry: + br i1 %flag, label %if.then, label %if.else + +if.then: + %a = load ptr, ptr %p + store ptr %p, ptr %a, align 8 + br label %if.end + +if.else: + %b = load ptr, ptr %p + store ptr %p, ptr %b, align 8, !nontemporal !1 + br label %if.end + +if.end: + ret void +} !0 = !{} +!1 = !{i32 1} diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll --- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll @@ -1559,3 +1559,47 @@ end: ret void } + +define void @nontemporal(ptr %ptr, i1 %cond) { +; CHECK-LABEL: @nontemporal( +; CHECK-NEXT: entry: +; CHECK-NEXT: store i64 0, ptr [[PTR:%.*]], align 8, !nontemporal !7 +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else + +if.then: + store i64 0, ptr %ptr, align 8, !nontemporal !12 + br label %if.end + +if.else: + store i64 0, ptr %ptr, align 8, !nontemporal !12 + br label %if.end + +if.end: + ret void +} + +define void @nontemporal_mismatch(ptr %ptr, i1 %cond) { +; CHECK-LABEL: @nontemporal_mismatch( +; CHECK-NEXT: entry: +; CHECK-NEXT: store i64 0, ptr [[PTR:%.*]], align 8 +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else + +if.then: + store i64 0, ptr %ptr, align 8, !nontemporal !12 + br label %if.end + +if.else: + store i64 0, ptr %ptr, align 8 + br label %if.end + +if.end: + ret void +} + +!12 = !{i32 1}