Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -12181,6 +12181,50 @@ arguments of the specified types, and not as varargs. +'``llvm.experimental.guard``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.experimental.guard(i1, ...) [ "deopt"(...) ] + +Overview: +""""""""" + +This intrinsic, together with :ref:`deoptimization operand bundles +`, allows frontends to express guards or checks on +optimistic assumptions made during compilation. The semantics of +``@llvm.experimental.guard`` is defined in terms of +``@llvm.experimental.deoptimize`` -- its body is defined to be +equivalent to: + +.. code-block:: llvm + + define void @llvm.experimental.guard(i1 %pred, ) { + %realPred = and i1 %pred, undef + br i1 %realPred, label %continue, label %leave + + leave: + call void @llvm.experimental.deoptimize() [ "deopt"() ] + ret void + + continue: + ret void + } + +In words, ``@llvm.experimental.guard`` executes the attached +``"deopt"`` continuation if (but **not** only if) its first argument +is ``false``. Since the optimizer is allowed to replace the ``undef`` +with an arbitrary value, it can optimize guard to fail "spuriously", +i.e. without the original condition being false (hence the "not only +if"); and this allows for "check widening" type optimizations. + +``@llvm.experimental.guard`` cannot be invoked. + + Stack Map Intrinsics -------------------- Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -597,6 +597,10 @@ def int_experimental_deoptimize : Intrinsic<[llvm_any_ty], [llvm_vararg_ty], [Throws]>; +// Support for speculative runtime guards +def int_experimental_guard : Intrinsic<[], [llvm_i1_ty, llvm_vararg_ty], + [Throws]>; + // NOP: calls/invokes to this intrinsic are removed by codegen def int_donothing : Intrinsic<[], [], [IntrNoMem]>; Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -188,6 +188,7 @@ void initializeLowerAtomicPass(PassRegistry&); void initializeLowerBitSetsPass(PassRegistry&); void initializeLowerExpectIntrinsicPass(PassRegistry&); +void initializeLowerGuardIntrinsicPass(PassRegistry&); void initializeLowerIntrinsicsPass(PassRegistry&); void initializeLowerInvokePass(PassRegistry&); void initializeLowerSwitchPass(PassRegistry&); Index: include/llvm/Transforms/Scalar.h =================================================================== --- include/llvm/Transforms/Scalar.h +++ include/llvm/Transforms/Scalar.h @@ -374,6 +374,12 @@ //===----------------------------------------------------------------------===// // +// LowerGuardIntrinsic - Lower guard intrinsics to normal control flow. +// +Pass *createLowerGuardIntrinsicPass(); + +//===----------------------------------------------------------------------===// +// // ValuePropagation - Propagate CFG-derived value information // Pass *createCorrelatedValuePropagationPass(); Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -4106,6 +4106,14 @@ break; } + case Intrinsic::experimental_guard: { + Assert(CS.isCall(), "experimental_guard cannot be invoked", CS); + Assert(CS.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, + "experimental_guard must have exactly one " + "\"deopt\" operand bundle"); + break; + } + case Intrinsic::experimental_deoptimize: { Assert(CS.isCall(), "experimental_deoptimize cannot be invoked", CS); Assert(CS.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, Index: lib/Transforms/Scalar/CMakeLists.txt =================================================================== --- lib/Transforms/Scalar/CMakeLists.txt +++ lib/Transforms/Scalar/CMakeLists.txt @@ -32,6 +32,7 @@ LoopVersioningLICM.cpp LowerAtomic.cpp LowerExpectIntrinsic.cpp + LowerGuardIntrinsic.cpp MemCpyOptimizer.cpp MergedLoadStoreMotion.cpp NaryReassociate.cpp Index: lib/Transforms/Scalar/LowerGuardIntrinsic.cpp =================================================================== --- /dev/null +++ lib/Transforms/Scalar/LowerGuardIntrinsic.cpp @@ -0,0 +1,107 @@ +//===- LowerGuardIntrinsic.cpp - Lower the guard intrinsic ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass lowers the llvm.experimental.guard intrinsic to a conditional call +// to @llvm.experimental.deoptimize. Once this happens, the guard can no longer +// be widened. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Scalar.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" + +using namespace llvm; + +namespace { +struct LowerGuardIntrinsic : public FunctionPass { + static char ID; + LowerGuardIntrinsic() : FunctionPass(ID) { + initializeLowerGuardIntrinsicPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; +}; +} + +static void MakeGuardControlFlowExplicit(Function *DeoptIntrinsic, + CallInst *CI) { + OperandBundleDef DeoptOB(*CI->getOperandBundle(LLVMContext::OB_deopt)); + SmallVector Args(std::next(CI->arg_begin()), CI->arg_end()); + + auto *CheckBB = CI->getParent(); + auto *DeoptBlockTerm = + SplitBlockAndInsertIfThen(CI->getArgOperand(0), CI, true); + + auto *CheckBI = cast(CheckBB->getTerminator()); + + // SplitBlockAndInsertIfThen inserts control flow that branches to + // DeoptBlockTerm if the condition is true. We want the opposite. + CheckBI->swapSuccessors(); + + CheckBI->getSuccessor(0)->setName("guarded"); + CheckBI->getSuccessor(1)->setName("deopt"); + + IRBuilder<> B(DeoptBlockTerm); + auto *DeoptCall = B.CreateCall(DeoptIntrinsic, Args, {DeoptOB}, ""); + + if (DeoptIntrinsic->getReturnType()->isVoidTy()) { + B.CreateRetVoid(); + } else { + DeoptCall->setName("deoptcall"); + B.CreateRet(DeoptCall); + } + + DeoptBlockTerm->eraseFromParent(); +} + +bool LowerGuardIntrinsic::runOnFunction(Function &F) { + if (!F.getParent()->getFunction( + Intrinsic::getName(Intrinsic::experimental_guard))) { + // No need to scan F, since we obviously have no calls to guard. + return false; + } + + SmallVector ToLower; + for (auto &I : instructions(F)) + if (auto *CI = dyn_cast(&I)) + if (auto *F = CI->getCalledFunction()) + if (F->getIntrinsicID() == Intrinsic::experimental_guard) + ToLower.push_back(CI); + + if (ToLower.empty()) + return false; + + auto *DeoptIntrinsic = Intrinsic::getDeclaration( + F.getParent(), Intrinsic::experimental_deoptimize, {F.getReturnType()}); + + for (auto *CI : ToLower) { + MakeGuardControlFlowExplicit(DeoptIntrinsic, CI); + CI->eraseFromParent(); + } + + return true; +} + +char LowerGuardIntrinsic::ID = 0; +INITIALIZE_PASS(LowerGuardIntrinsic, "lower-guard-intrinsic", + "Lower the guard intrinsic to normal control flow", false, + false) + +Pass *llvm::createLowerGuardIntrinsicPass() { + return new LowerGuardIntrinsic(); +} Index: lib/Transforms/Scalar/Scalar.cpp =================================================================== --- lib/Transforms/Scalar/Scalar.cpp +++ lib/Transforms/Scalar/Scalar.cpp @@ -62,6 +62,7 @@ initializeLoopIdiomRecognizePass(Registry); initializeLowerAtomicPass(Registry); initializeLowerExpectIntrinsicPass(Registry); + initializeLowerGuardIntrinsicPass(Registry); initializeMemCpyOptPass(Registry); initializeMergedLoadStoreMotionPass(Registry); initializeNaryReassociatePass(Registry); Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -428,12 +428,14 @@ continue; // We do not need to (and in fact, cannot) convert possibly throwing calls - // to @llvm.experimental_deoptimize into invokes. The caller's "segment" of - // the deoptimization continuation attached to the newly inlined - // @llvm.experimental_deoptimize call should contain the exception handling - // logic, if any. + // to @llvm.experimental_deoptimize (resp. @llvm.experimental.guard) into + // invokes. The caller's "segment" of the deoptimization continuation + // attached to the newly inlined @llvm.experimental_deoptimize + // (resp. @llvm.experimental.guard) call should contain the exception + // handling logic, if any. if (auto *F = CI->getCalledFunction()) - if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) + if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize || + F->getIntrinsicID() == Intrinsic::experimental_guard) continue; if (auto FuncletBundle = CI->getOperandBundle(LLVMContext::OB_funclet)) { Index: test/Transforms/Inline/guard-intrinsic.ll =================================================================== --- /dev/null +++ test/Transforms/Inline/guard-intrinsic.ll @@ -0,0 +1,39 @@ +; RUN: opt -S -always-inline < %s | FileCheck %s + +declare void @llvm.experimental.guard(i1, ...) + +define i8 @callee(i1* %c_ptr) alwaysinline { + %c = load volatile i1, i1* %c_ptr + call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1) [ "deopt"(i32 1) ] + ret i8 5 +} + +define void @caller_0(i1* %c, i8* %ptr) { +; CHECK-LABEL: @caller_0( +entry: +; CHECK: [[COND:%[^ ]+]] = load volatile i1, i1* %c +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[COND]], i32 1) [ "deopt"(i32 2, i32 1) ] +; CHECK-NEXT: store i8 5, i8* %ptr + + %v = call i8 @callee(i1* %c) [ "deopt"(i32 2) ] + store i8 %v, i8* %ptr + ret void +} + +define i32 @caller_1(i1* %c, i8* %ptr) personality i8 3 { +; CHECK-LABEL: @caller_1( +; CHECK: [[COND:%[^ ]+]] = load volatile i1, i1* %c +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[COND]], i32 1) [ "deopt"(i32 3, i32 1) ] +; CHECK-NEXT: br label %normal +entry: + %v = invoke i8 @callee(i1* %c) [ "deopt"(i32 3) ] to label %normal + unwind label %unwind + +unwind: + %lp = landingpad i32 cleanup + ret i32 43 + +normal: + store i8 %v, i8* %ptr + ret i32 42 +} Index: test/Transforms/LowerGuardIntrinsic/basic.ll =================================================================== --- /dev/null +++ test/Transforms/LowerGuardIntrinsic/basic.ll @@ -0,0 +1,62 @@ +; RUN: opt -S -lower-guard-intrinsic < %s | FileCheck %s + +declare void @llvm.experimental.guard(i1, ...) + +define i8 @f_basic(i1* %c_ptr) { +; CHECK-LABEL: @f_basic( + + %c = load volatile i1, i1* %c_ptr + call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1) [ "deopt"(i32 1) ] + ret i8 5 + +; CHECK: br i1 %c, label %guarded, label %deopt +; CHECK: deopt: +; CHECK-NEXT: %deoptcall = call i8 (...) @llvm.experimental.deoptimize.i8(i32 1) [ "deopt"(i32 1) ] +; CHECK-NEXT: ret i8 %deoptcall +; CHECK: guarded: +; CHECK-NEXT: ret i8 5 +} + +define void @f_void_return_ty(i1* %c_ptr) { +; CHECK-LABEL: @f_void_return_ty( + + %c = load volatile i1, i1* %c_ptr + call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1) [ "deopt"() ] + ret void + +; CHECK: br i1 %c, label %guarded, label %deopt +; CHECK: deopt: +; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid(i32 1) [ "deopt"() ] +; CHECK-NEXT: ret void +; CHECK: guarded: +; CHECK-NEXT: ret void +} + +define void @f_multiple_args(i1* %c_ptr) { +; CHECK-LABEL: @f_multiple_args( + + %c = load volatile i1, i1* %c_ptr + call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1, i32 2, double 500.0) [ "deopt"(i32 2, i32 3) ] + ret void + +; CHECK: br i1 %c, label %guarded, label %deopt +; CHECK: deopt: +; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid(i32 1, i32 2, double 5.000000e+02) [ "deopt"(i32 2, i32 3) ] +; CHECK-NEXT: ret void +; CHECK: guarded: +; CHECK-NEXT: ret void +} + +define i32 @f_zero_args(i1* %c_ptr) { +; CHECK-LABEL: @f_zero_args( + %c = load volatile i1, i1* %c_ptr + call void(i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"(i32 2, i32 3) ] + ret i32 500 + +; CHECK: br i1 %c, label %guarded, label %deopt +; CHECK: deopt: +; CHECK-NEXT: %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 2, i32 3) ] +; CHECK-NEXT: ret i32 %deoptcall +; CHECK: guarded: +; CHECK-NEXT: ret i32 500 +} Index: test/Verifier/guard-intrinsic.ll =================================================================== --- /dev/null +++ test/Verifier/guard-intrinsic.ll @@ -0,0 +1,26 @@ +; RUN: not opt -S -verify < %s 2>&1 | FileCheck %s + +declare void @llvm.experimental.guard(i1, ...) + +declare void @unknown() + +define void @f_nodeopt() { +entry: + call void(i1, ...) @llvm.experimental.guard(i1 undef, i32 1, i32 2) +; CHECK: guard must have exactly one "deopt" operand bundle + ret void +} + +define void @f_invoke() personality i8 3 { +entry: + invoke void(i1, ...) @llvm.experimental.guard(i1 undef, i32 0, float 0.0) [ "deopt"() ] to label %ok unwind label %not_ok +; CHECK: guard cannot be invoked + +ok: + ret void + +not_ok: + %0 = landingpad { i8*, i32 } + filter [0 x i8*] zeroinitializer + ret void +}