Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -14177,6 +14177,36 @@ a constant initializer folded into a function body. This intrinsic can be used to avoid the possibility of overflows when loading from such a constant. +'``llvm.sideeffect``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.sideeffect() inaccessiblememonly nounwind + +Overview: +""""""""" + +The ``llvm.sideeffect`` intrinsic doesn't perform any operation. Optimizers +treat it as having side effects, so it can be inserted into a loop to +indicate that the loop shouldn't be optimized away even if it's an infinite +loop with no other side effects. + +Arguments: +"""""""""" + +None. + +Semantics: +"""""""""" + +This intrinsic actually does nothing, but optimizers must assume that it +has externally observable side effects and therefore may not be deleted +or reordered past other externally observable side effects. + Stack Map Intrinsics -------------------- Index: include/llvm/Analysis/TargetTransformInfoImpl.h =================================================================== --- include/llvm/Analysis/TargetTransformInfoImpl.h +++ include/llvm/Analysis/TargetTransformInfoImpl.h @@ -152,6 +152,7 @@ case Intrinsic::annotation: case Intrinsic::assume: + case Intrinsic::sideeffect: case Intrinsic::dbg_declare: case Intrinsic::dbg_value: case Intrinsic::invariant_start: Index: include/llvm/CodeGen/BasicTTIImpl.h =================================================================== --- include/llvm/CodeGen/BasicTTIImpl.h +++ include/llvm/CodeGen/BasicTTIImpl.h @@ -988,6 +988,7 @@ // FIXME: We should return 0 whenever getIntrinsicCost == TCC_Free. case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: + case Intrinsic::sideeffect: return 0; case Intrinsic::masked_store: return static_cast(this) Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -810,6 +810,12 @@ // NOP: calls/invokes to this intrinsic are removed by codegen def int_donothing : Intrinsic<[], [], [IntrNoMem]>; +// This instruction has no actual effect, though it is treated by the optimizer +// has having opaque side effects. This may be inserted into loops to ensure +// that they are not removed even if they turn out to be empty, for languages +// which specify that infinite loops must be preserved. +def int_sideeffect : Intrinsic<[], [], [IntrInaccessibleMemOnly]>; + // Intrisics to support half precision floating point format let IntrProperties = [IntrNoMem] in { def int_convert_to_fp16 : Intrinsic<[llvm_i16_ty], [llvm_anyfloat_ty]>; Index: lib/Analysis/AliasSetTracker.cpp =================================================================== --- lib/Analysis/AliasSetTracker.cpp +++ lib/Analysis/AliasSetTracker.cpp @@ -436,6 +436,7 @@ break; // FIXME: Add lifetime/invariant intrinsics (See: PR30807). case Intrinsic::assume: + case Intrinsic::sideeffect: return; } } Index: lib/Analysis/ValueTracking.cpp =================================================================== --- lib/Analysis/ValueTracking.cpp +++ lib/Analysis/ValueTracking.cpp @@ -439,6 +439,7 @@ default: break; // FIXME: This list is repeated from NoTTI::getIntrinsicCost. case Intrinsic::assume: + case Intrinsic::sideeffect: case Intrinsic::dbg_declare: case Intrinsic::dbg_value: case Intrinsic::invariant_start: @@ -3873,7 +3874,8 @@ // FIXME: This isn't aggressive enough; a call which only writes to a global // is guaranteed to return. return CS.onlyReadsMemory() || CS.onlyAccessesArgMemory() || - match(I, m_Intrinsic()); + match(I, m_Intrinsic()) || + match(I, m_Intrinsic()); } // Other instructions return normally. Index: lib/Analysis/VectorUtils.cpp =================================================================== --- lib/Analysis/VectorUtils.cpp +++ lib/Analysis/VectorUtils.cpp @@ -91,7 +91,8 @@ return Intrinsic::not_intrinsic; if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start || - ID == Intrinsic::lifetime_end || ID == Intrinsic::assume) + ID == Intrinsic::lifetime_end || ID == Intrinsic::assume || + ID == Intrinsic::sideeffect) return ID; return Intrinsic::not_intrinsic; } Index: lib/CodeGen/SelectionDAG/FastISel.cpp =================================================================== --- lib/CodeGen/SelectionDAG/FastISel.cpp +++ lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1133,6 +1133,8 @@ case Intrinsic::lifetime_end: // The donothing intrinsic does, well, nothing. case Intrinsic::donothing: + // Neither does the sideeffect intrinsic. + case Intrinsic::sideeffect: // Neither does the assume intrinsic; it's also OK not to codegen its operand. case Intrinsic::assume: return true; Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5662,7 +5662,8 @@ return nullptr; case Intrinsic::assume: case Intrinsic::var_annotation: - // Discard annotate attributes and assumptions + case Intrinsic::sideeffect: + // Discard annotate attributes, assumptions, and artificial side-effects. return nullptr; case Intrinsic::codeview_annotation: { Index: lib/Transforms/Scalar/EarlyCSE.cpp =================================================================== --- lib/Transforms/Scalar/EarlyCSE.cpp +++ lib/Transforms/Scalar/EarlyCSE.cpp @@ -663,6 +663,12 @@ continue; } + // Skip sideeffect intrinsics, for the same reason as assume intrinsics. + if (match(Inst, m_Intrinsic())) { + DEBUG(dbgs() << "EarlyCSE skipping sideeffect: " << *Inst << '\n'); + continue; + } + // Skip invariant.start intrinsics since they only read memory, and we can // forward values across it. Also, we dont need to consume the last store // since the semantics of invariant.start allow us to perform DSE of the Index: lib/Transforms/Scalar/GVNHoist.cpp =================================================================== --- lib/Transforms/Scalar/GVNHoist.cpp +++ lib/Transforms/Scalar/GVNHoist.cpp @@ -1079,7 +1079,8 @@ else if (auto *Call = dyn_cast(&I1)) { if (auto *Intr = dyn_cast(Call)) { if (isa(Intr) || - Intr->getIntrinsicID() == Intrinsic::assume) + Intr->getIntrinsicID() == Intrinsic::assume || + Intr->getIntrinsicID() == Intrinsic::sideeffect) continue; } if (Call->mayHaveSideEffects()) Index: lib/Transforms/Utils/Evaluator.cpp =================================================================== --- lib/Transforms/Utils/Evaluator.cpp +++ lib/Transforms/Utils/Evaluator.cpp @@ -420,6 +420,10 @@ DEBUG(dbgs() << "Skipping assume intrinsic.\n"); ++CurInst; continue; + } else if (II->getIntrinsicID() == Intrinsic::sideeffect) { + DEBUG(dbgs() << "Skipping sideeffect intrinsic.\n"); + ++CurInst; + continue; } DEBUG(dbgs() << "Unknown intrinsic. Can not evaluate.\n"); Index: lib/Transforms/Vectorize/LoadStoreVectorizer.cpp =================================================================== --- lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -25,6 +25,7 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -480,6 +481,10 @@ MemoryInstrs.push_back(&I); else ChainInstrs.push_back(&I); + } else if (isa(&I) && + cast(&I)->getIntrinsicID() == + Intrinsic::sideeffect) { + // Ignore llvm.sideeffect calls. } else if (IsLoadChain && (I.mayWriteToMemory() || I.mayThrow())) { DEBUG(dbgs() << "LSV: Found may-write/throw operation: " << I << '\n'); break; Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8045,7 +8045,7 @@ if (CallInst *CI = dyn_cast(I)) { Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end || - ID == Intrinsic::lifetime_start)) + ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect)) return nullptr; } Index: lib/Transforms/Vectorize/SLPVectorizer.cpp =================================================================== --- lib/Transforms/Vectorize/SLPVectorizer.cpp +++ lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3601,7 +3601,9 @@ "new ScheduleData already in scheduling region"); SD->init(SchedulingRegionID, I); - if (I->mayReadOrWriteMemory()) { + if (I->mayReadOrWriteMemory() && + (!isa(I) || + cast(I)->getIntrinsicID() != Intrinsic::sideeffect)) { // Update the linked list of memory accessing instructions. if (CurrentLoadStore) { CurrentLoadStore->NextLoadStore = SD; Index: test/CodeGen/Generic/intrinsics.ll =================================================================== --- test/CodeGen/Generic/intrinsics.ll +++ test/CodeGen/Generic/intrinsics.ll @@ -45,3 +45,12 @@ %q = call i8* @llvm.invariant.group.barrier(i8* %p) ret i8* %q } + +; sideeffect + +declare void @llvm.sideeffect() + +define void @test_sideeffect() { + call void @llvm.sideeffect() + ret void +} Index: test/Transforms/DCE/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/DCE/int_sideeffect.ll @@ -0,0 +1,12 @@ +; RUN: opt -S < %s -instcombine | FileCheck %s + +declare void @llvm.sideeffect() + +; Don't DCE llvm.sideeffect calls. + +; CHECK-LABEL: dce +; CHECK: call void @llvm.sideeffect() +define void @dce() { + call void @llvm.sideeffect() + ret void +} Index: test/Transforms/DeadStoreElimination/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/DeadStoreElimination/int_sideeffect.ll @@ -0,0 +1,15 @@ +; RUN: opt -S < %s -dse | FileCheck %s + +declare void @llvm.sideeffect() + +; Dead store elimination across a @llvm.sideeffect. + +; CHECK-LABEL: dse +; CHECK: store +; CHECK-NOT: store +define void @dse(float* %p) { + store float 0.0, float* %p + call void @llvm.sideeffect() + store float 0.0, float* %p + ret void +} Index: test/Transforms/EarlyCSE/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/EarlyCSE/int_sideeffect.ll @@ -0,0 +1,27 @@ +; RUN: opt -S < %s -early-cse | FileCheck %s + +declare void @llvm.sideeffect() + +; Store-to-load forwarding across a @llvm.sideeffect. + +; CHECK-LABEL: s2l +; CHECK-NOT: load +define float @s2l(float* %p) { + store float 0.0, float* %p + call void @llvm.sideeffect() + %t = load float, float* %p + ret float %t +} + +; Redundant load elimination across a @llvm.sideeffect. + +; CHECK-LABEL: rle +; CHECK: load +; CHECK-NOT: load +define float @rle(float* %p) { + %r = load float, float* %p + call void @llvm.sideeffect() + %s = load float, float* %p + %t = fadd float %r, %s + ret float %t +} Index: test/Transforms/FunctionAttrs/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/FunctionAttrs/int_sideeffect.ll @@ -0,0 +1,21 @@ +; RUN: opt -S < %s -functionattrs | FileCheck %s + +declare void @llvm.sideeffect() + +; Don't add readnone or similar attributes when an @llvm.sideeffect() intrinsic +; is present. + +; CHECK: define void @test() { +define void @test() { + call void @llvm.sideeffect() + ret void +} + +; CHECK: define void @loop() { +define void @loop() { + br label %loop + +loop: + call void @llvm.sideeffect() + br label %loop +} Index: test/Transforms/GVN/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/GVN/int_sideeffect.ll @@ -0,0 +1,51 @@ +; RUN: opt -S < %s -gvn | FileCheck %s + +declare void @llvm.sideeffect() + +; Store-to-load forwarding across a @llvm.sideeffect. + +; CHECK-LABEL: s2l +; CHECK-NOT: load +define float @s2l(float* %p) { + store float 0.0, float* %p + call void @llvm.sideeffect() + %t = load float, float* %p + ret float %t +} + +; Redundant load elimination across a @llvm.sideeffect. + +; CHECK-LABEL: rle +; CHECK: load +; CHECK-NOT: load +define float @rle(float* %p) { + %r = load float, float* %p + call void @llvm.sideeffect() + %s = load float, float* %p + %t = fadd float %r, %s + ret float %t +} + +; LICM across a @llvm.sideeffect. + +; CHECK-LABEL: licm +; CHECK: load +; CHECK: loop: +; CHECK-NOT: load +define float @licm(i64 %n, float* nocapture readonly %p) #0 { +bb0: + br label %loop + +loop: + %i = phi i64 [ 0, %bb0 ], [ %t5, %loop ] + %sum = phi float [ 0.000000e+00, %bb0 ], [ %t4, %loop ] + call void @llvm.sideeffect() + %t3 = load float, float* %p + %t4 = fadd float %sum, %t3 + %t5 = add i64 %i, 1 + %t6 = icmp ult i64 %t5, %n + br i1 %t6, label %loop, label %bb2 + +bb2: + ret float %t4 +} Index: test/Transforms/GVNHoist/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/GVNHoist/int_sideeffect.ll @@ -0,0 +1,30 @@ +; RUN: opt -S < %s -gvn-hoist | FileCheck %s + +declare void @llvm.sideeffect() + +; GVN hoisting across a @llvm.sideeffect. + +; CHECK-LABEL: scalarsHoisting +; CHECK: = fsub +; CHECK: br i1 %cmp, +; CHECK-NOT: fsub +define float @scalarsHoisting(float %d, float %m, float %a, i1 %cmp) { +entry: + br i1 %cmp, label %if.then, label %if.else + +if.then: + call void @llvm.sideeffect() + %sub0 = fsub float %m, %a + %mul = fmul float %sub0, %d + br label %if.end + +if.else: + %sub1 = fsub float %m, %a + %div = fdiv float %sub1, %d + br label %if.end + +if.end: + %phi = phi float [ %mul, %if.then ], [ %div, %if.else ] + ret float %phi +} + Index: test/Transforms/GVNSink/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/GVNSink/int_sideeffect.ll @@ -0,0 +1,30 @@ +; RUN: opt -S < %s -gvn-sink | FileCheck %s + +declare void @llvm.sideeffect() + +; GVN sinking across a @llvm.sideeffect. + +; CHECK-LABEL: scalarsSinking +; CHECK-NOT: fmul +; CHECK: = phi +; CHECK: = fmul +define float @scalarsSinking(float %d, float %m, float %a, i1 %cmp) { +entry: + br i1 %cmp, label %if.then, label %if.else + +if.then: + call void @llvm.sideeffect() + %sub = fsub float %m, %a + %mul0 = fmul float %sub, %d + br label %if.end + +if.else: + %add = fadd float %m, %a + %mul1 = fmul float %add, %d + br label %if.end + +if.end: + %phi = phi float [ %mul0, %if.then ], [ %mul1, %if.else ] + ret float %phi +} + Index: test/Transforms/GlobalOpt/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/GlobalOpt/int_sideeffect.ll @@ -0,0 +1,16 @@ +; RUN: opt -S < %s -globalopt | FileCheck %s + +; Static evaluation across a @llvm.sideeffect. + +; CHECK-NOT: store + +declare void @llvm.sideeffect() + +@llvm.global_ctors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @ctor } ] +@G = global i32 0 + +define internal void @ctor() { + store i32 1, i32* @G + call void @llvm.sideeffect() + ret void +} Index: test/Transforms/InstCombine/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/InstCombine/int_sideeffect.ll @@ -0,0 +1,14 @@ +; RUN: opt -S < %s -instcombine | FileCheck %s + +declare void @llvm.sideeffect() + +; Store-to-load forwarding across a @llvm.sideeffect. + +; CHECK-LABEL: s2l +; CHECK-NOT: load +define float @s2l(float* %p) { + store float 0.0, float* %p + call void @llvm.sideeffect() + %t = load float, float* %p + ret float %t +} Index: test/Transforms/LICM/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/LICM/int_sideeffect.ll @@ -0,0 +1,27 @@ +; RUN: opt -S < %s -licm | FileCheck %s + +declare void @llvm.sideeffect() + +; LICM across a @llvm.sideeffect. + +; CHECK-LABEL: licm +; CHECK: load +; CHECK: loop: +; CHECK-NOT: load +define float @licm(i64 %n, float* nocapture readonly %p) #0 { +bb0: + br label %loop + +loop: + %i = phi i64 [ 0, %bb0 ], [ %t5, %loop ] + %sum = phi float [ 0.000000e+00, %bb0 ], [ %t4, %loop ] + call void @llvm.sideeffect() + %t3 = load float, float* %p + %t4 = fadd float %sum, %t3 + %t5 = add i64 %i, 1 + %t6 = icmp ult i64 %t5, %n + br i1 %t6, label %loop, label %bb2 + +bb2: + ret float %t4 +} Index: test/Transforms/LoadStoreVectorizer/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/LoadStoreVectorizer/int_sideeffect.ll @@ -0,0 +1,26 @@ +; RUN: opt -S < %s -load-store-vectorizer | FileCheck %s + +declare void @llvm.sideeffect() + +; load-store vectorization across a @llvm.sideeffect. + +; CHECK-LABEL: test +; CHECK: load <4 x float> +; CHECK: store <4 x float> +define void @test(float* %p) { + %p0 = getelementptr float, float* %p, i64 0 + %p1 = getelementptr float, float* %p, i64 1 + %p2 = getelementptr float, float* %p, i64 2 + %p3 = getelementptr float, float* %p, i64 3 + %l0 = load float, float* %p0, align 16 + %l1 = load float, float* %p1 + %l2 = load float, float* %p2 + call void @llvm.sideeffect() + %l3 = load float, float* %p3 + store float %l0, float* %p0, align 16 + call void @llvm.sideeffect() + store float %l1, float* %p1 + store float %l2, float* %p2 + store float %l3, float* %p3 + ret void +} Index: test/Transforms/LoopIdiom/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/LoopIdiom/int_sideeffect.ll @@ -0,0 +1,23 @@ +; RUN: opt -S < %s -loop-idiom | FileCheck %s + +declare void @llvm.sideeffect() + +; Loop idiom recognition across a @llvm.sideeffect. + +; CHECK-LABEL: zero +; CHECK: llvm.memset +define void @zero(float* %p, i64 %n) nounwind { +bb7.lr.ph: + br label %bb7 + +bb7: + %i.02 = phi i64 [ 0, %bb7.lr.ph ], [ %tmp13, %bb7 ] + %tmp10 = getelementptr inbounds float, float* %p, i64 %i.02 + store float 0.000000e+00, float* %tmp10, align 4 + %tmp13 = add i64 %i.02, 1 + %tmp6 = icmp ult i64 %tmp13, %n + br i1 %tmp6, label %bb7, label %bb14 + +bb14: + ret void +} Index: test/Transforms/LoopVectorize/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/LoopVectorize/int_sideeffect.ll @@ -0,0 +1,24 @@ +; RUN: opt -S < %s -loop-vectorize -force-vector-width=4 | FileCheck %s + +declare void @llvm.sideeffect() + +; Vectorization across a @llvm.sideeffect. + +; CHECK-LABEL: store_ones +; CHECK: store <4 x float> +define void @store_ones(float* %p, i64 %n) nounwind { +bb7.lr.ph: + br label %bb7 + +bb7: + %i.02 = phi i64 [ 0, %bb7.lr.ph ], [ %tmp13, %bb7 ] + call void @llvm.sideeffect() + %tmp10 = getelementptr inbounds float, float* %p, i64 %i.02 + store float 1.0, float* %tmp10, align 4 + %tmp13 = add i64 %i.02, 1 + %tmp6 = icmp ult i64 %tmp13, %n + br i1 %tmp6, label %bb7, label %bb14 + +bb14: + ret void +} Index: test/Transforms/NewGVN/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/NewGVN/int_sideeffect.ll @@ -0,0 +1,27 @@ +; RUN: opt -S < %s -newgvn | FileCheck %s + +declare void @llvm.sideeffect() + +; Store-to-load forwarding across a @llvm.sideeffect. + +; CHECK-LABEL: s2l +; CHECK-NOT: load +define float @s2l(float* %p) { + store float 0.0, float* %p + call void @llvm.sideeffect() + %t = load float, float* %p + ret float %t +} + +; Redundant load elimination across a @llvm.sideeffect. + +; CHECK-LABEL: rle +; CHECK: load +; CHECK-NOT: load +define float @rle(float* %p) { + %r = load float, float* %p + call void @llvm.sideeffect() + %s = load float, float* %p + %t = fadd float %r, %s + ret float %t +} Index: test/Transforms/SLPVectorizer/int_sideeffect.ll =================================================================== --- /dev/null +++ test/Transforms/SLPVectorizer/int_sideeffect.ll @@ -0,0 +1,25 @@ +; RUN: opt -S < %s -slp-vectorizer -slp-max-reg-size=128 -slp-min-reg-size=128 | FileCheck %s + +declare void @llvm.sideeffect() + +; SLP vectorization across a @llvm.sideeffect. + +; CHECK-LABEL: test +; CHECK: store <4 x float> +define void @test(float* %p) { + %p0 = getelementptr float, float* %p, i64 0 + %p1 = getelementptr float, float* %p, i64 1 + %p2 = getelementptr float, float* %p, i64 2 + %p3 = getelementptr float, float* %p, i64 3 + %l0 = load float, float* %p0 + %l1 = load float, float* %p1 + %l2 = load float, float* %p2 + call void @llvm.sideeffect() + %l3 = load float, float* %p3 + store float %l0, float* %p0 + call void @llvm.sideeffect() + store float %l1, float* %p1 + store float %l2, float* %p2 + store float %l3, float* %p3 + ret void +}