Index: bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli =================================================================== --- bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli +++ bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli @@ -177,10 +177,10 @@ : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit = "llvm_add_early_cse" -(** See the [llvm::createLowerExpectIntrinsicPass] function. *) -external add_lower_expect_intrinsic +(** See the [llvm::createLowerPredictionIntrinsicPass] function. *) +external add_lower_prediction_intrinsic : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit - = "llvm_add_lower_expect_intrinsic" + = "llvm_add_lower_prediction_intrinsic" (** See the [llvm::createTypeBasedAliasAnalysisPass] function. *) external add_type_based_alias_analysis Index: bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c =================================================================== --- bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c +++ bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c @@ -219,8 +219,8 @@ } /* [ unit */ -CAMLprim value llvm_add_lower_expect_intrinsic(LLVMPassManagerRef PM) { - LLVMAddLowerExpectIntrinsicPass(PM); +CAMLprim value llvm_add_lower_prediction_intrinsic(LLVMPassManagerRef PM) { + LLVMAddLowerPredictionIntrinsicPass(PM); return Val_unit; } Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -11656,7 +11656,7 @@ Overview: """"""""" -The ``llvm.expect`` intrinsic provides information about expected (the +The ``llvm.expect`` intrinsic provides information about the expected (the most probable) value of ``val``, which can be used by optimizers. Arguments: @@ -11671,6 +11671,42 @@ This intrinsic is lowered to the ``val``. +'``llvm.unpredictable``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +This is an overloaded intrinsic. You can use ``llvm.unpredictable`` on any +integer bit width. + +:: + + declare i1 @llvm.unpredictable.i1(i1 ) + declare i32 @llvm.unpredictable.i32(i32 ) + declare i64 @llvm.unpredictable.i64(i64 ) + +Overview: +""""""""" + +The ``llvm.unpredictable`` intrinsic indicates that ``val`` will not be +predicted correctly by hardware mechanisms such as branch predictors. It is +expected that ``val`` will be used an input argument to a branch or switch +instruction. + +Arguments: +"""""""""" + +The ``llvm.unpredictable`` intrinsic takes one argument. The argument is +a value that is unlikely to be predicted correctly dynamically. + +Semantics: +"""""""""" + +This intrinsic is lowered to the ``val`` and may be replaced by metadata +that can be used by an IR or backend optimizer. This intrinsic provides +the inverse functionality of the ``llvm.expect`` intrinsic. + .. _int_assume: '``llvm.assume``' Intrinsic Index: include/llvm-c/Transforms/Scalar.h =================================================================== --- include/llvm-c/Transforms/Scalar.h +++ include/llvm-c/Transforms/Scalar.h @@ -135,8 +135,8 @@ /** See llvm::createEarlyCSEPass function */ void LLVMAddEarlyCSEPass(LLVMPassManagerRef PM); -/** See llvm::createLowerExpectIntrinsicPass function */ -void LLVMAddLowerExpectIntrinsicPass(LLVMPassManagerRef PM); +/** See llvm::createLowerPredictionIntrinsicPass function */ +void LLVMAddLowerPredictionIntrinsicPass(LLVMPassManagerRef PM); /** See llvm::createTypeBasedAliasAnalysisPass function */ void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM); Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -385,10 +385,14 @@ [IntrNoMem]>, GCCBuiltin<"__builtin_object_size">; -//===------------------------- Expect Intrinsics --------------------------===// +//===------------------ Expect/Unpredictable Intrinsics -------------------===// // -def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, - LLVMMatchType<0>], [IntrNoMem]>; + +let Properties = [IntrNoMem] in { + def int_expect : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>]>; + def int_unpredictable : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; +} //===-------------------- Bit Manipulation Intrinsics ---------------------===// // Index: include/llvm/IR/LLVMContext.h =================================================================== --- include/llvm/IR/LLVMContext.h +++ include/llvm/IR/LLVMContext.h @@ -61,7 +61,8 @@ MD_nonnull = 11, // "nonnull" MD_dereferenceable = 12, // "dereferenceable" MD_dereferenceable_or_null = 13, // "dereferenceable_or_null" - MD_make_implicit = 14 // "make.implicit" + MD_make_implicit = 14, // "make.implicit" + MD_unpredictable = 15 // "unpredictable" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -174,7 +174,7 @@ void initializeLoopIdiomRecognizePass(PassRegistry&); void initializeLowerAtomicPass(PassRegistry&); void initializeLowerBitSetsPass(PassRegistry&); -void initializeLowerExpectIntrinsicPass(PassRegistry&); +void initializeLowerPredictionIntrinsicPass(PassRegistry&); void initializeLowerIntrinsicsPass(PassRegistry&); void initializeLowerInvokePass(PassRegistry&); void initializeLowerSwitchPass(PassRegistry&); Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -111,7 +111,7 @@ (void) llvm::createLoopUnswitchPass(); (void) llvm::createLoopIdiomPass(); (void) llvm::createLoopRotatePass(); - (void) llvm::createLowerExpectIntrinsicPass(); + (void) llvm::createLowerPredictionIntrinsicPass(); (void) llvm::createLowerInvokePass(); (void) llvm::createLowerSwitchPass(); (void) llvm::createNaryReassociatePass(); Index: include/llvm/Transforms/Scalar.h =================================================================== --- include/llvm/Transforms/Scalar.h +++ include/llvm/Transforms/Scalar.h @@ -395,9 +395,9 @@ //===----------------------------------------------------------------------===// // -// LowerExpectIntrinsics - Removes llvm.expect intrinsics and creates -// "block_weights" metadata. -FunctionPass *createLowerExpectIntrinsicPass(); +// LowerPredictionIntrinsics - Replaces llvm.expect and llvm.unpredictable +// intrinsics with metadata. +FunctionPass *createLowerPredictionIntrinsicPass(); //===----------------------------------------------------------------------===// // Index: include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h =================================================================== --- include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h +++ include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h @@ -1,40 +0,0 @@ -//===- LowerExpectIntrinsic.h - LowerExpectIntrinsic pass -------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// \file -/// -/// The header file for the LowerExpectIntrinsic pass as used by the new pass -/// manager. -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H -#define LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H - -#include "llvm/IR/Function.h" -#include "llvm/IR/PassManager.h" - -namespace llvm { - -class LowerExpectIntrinsicPass { -public: - static StringRef name() { return "LowerExpectIntrinsicPass"; } - - /// \brief Run the pass over the function. - /// - /// This will lower all of th expect intrinsic calls in this function into - /// branch weight metadata. That metadata will subsequently feed the analysis - /// of the probabilities and frequencies of the CFG. After running this pass, - /// no more expect intrinsics remain, allowing the rest of the optimizer to - /// ignore them. - PreservedAnalyses run(Function &F); -}; - -} - -#endif Index: include/llvm/Transforms/Scalar/LowerPredictionIntrinsic.h =================================================================== --- include/llvm/Transforms/Scalar/LowerPredictionIntrinsic.h +++ include/llvm/Transforms/Scalar/LowerPredictionIntrinsic.h @@ -1,4 +1,4 @@ -//===- LowerExpectIntrinsic.h - LowerExpectIntrinsic pass -------*- C++ -*-===// +//==- LowerPredictionIntrinsic.h - LowerPredictionIntrinsic pass -*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -8,30 +8,30 @@ //===----------------------------------------------------------------------===// /// \file /// -/// The header file for the LowerExpectIntrinsic pass as used by the new pass -/// manager. +/// The header file for the LowerPredictionIntrinsic pass as used by the new +/// pass manager. /// //===----------------------------------------------------------------------===// -#ifndef LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H -#define LLVM_TRANSFORMS_SCALAR_LOWEREXPECTINTRINSIC_H +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERPREDICTIONINTRINSIC_H +#define LLVM_TRANSFORMS_SCALAR_LOWERPREDICTIONINTRINSIC_H #include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" namespace llvm { -class LowerExpectIntrinsicPass { +class LowerPredictionIntrinsicPass { public: - static StringRef name() { return "LowerExpectIntrinsicPass"; } + static StringRef name() { return "LowerPredictionIntrinsicPass"; } /// \brief Run the pass over the function. /// - /// This will lower all of th expect intrinsic calls in this function into - /// branch weight metadata. That metadata will subsequently feed the analysis - /// of the probabilities and frequencies of the CFG. After running this pass, - /// no more expect intrinsics remain, allowing the rest of the optimizer to - /// ignore them. + /// This will lower expect and unpredictable intrinsics in this function into + /// metadata. That metadata will subsequently feed the analysis of the + /// probabilities and frequencies of the CFG. After running this pass, no + /// more expect or unpredictable intrinsics remain, allowing the rest of the + /// optimizer to ignore them. PreservedAnalyses run(Function &F); }; Index: lib/CodeGen/IntrinsicLowering.cpp =================================================================== --- lib/CodeGen/IntrinsicLowering.cpp +++ lib/CodeGen/IntrinsicLowering.cpp @@ -353,8 +353,10 @@ report_fatal_error("Code generator does not support intrinsic function '"+ Callee->getName()+"'!"); + case Intrinsic::unpredictable: case Intrinsic::expect: { - // Just replace __builtin_expect(exp, c) with EXP. + // Replace __builtin_expect(exp, c) or __builtin_unpredictable(exp) with + // exp. Value *V = CI->getArgOperand(0); CI->replaceAllUsesWith(V); break; Index: lib/CodeGen/SelectionDAG/FastISel.cpp =================================================================== --- lib/CodeGen/SelectionDAG/FastISel.cpp +++ lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1230,6 +1230,7 @@ updateValueMap(II, ResultReg); return true; } + case Intrinsic::unpredictable: case Intrinsic::expect: { unsigned ResultReg = getRegForValue(II->getArgOperand(0)); if (!ResultReg) Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4868,8 +4868,10 @@ setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32)); return nullptr; + case Intrinsic::unpredictable: case Intrinsic::expect: { - // Just replace __builtin_expect(exp, c) with EXP. + // Replace __builtin_expect(exp, c) or __builtin_unpredictable(exp) with + // exp. setValue(&I, getValue(I.getArgOperand(0))); return nullptr; } Index: lib/IR/LLVMContext.cpp =================================================================== --- lib/IR/LLVMContext.cpp +++ lib/IR/LLVMContext.cpp @@ -110,6 +110,12 @@ assert(MakeImplicitID == MD_make_implicit && "make.implicit kind id drifted"); (void)MakeImplicitID; + + // Create the 'unpredictable' metadata kind. + unsigned UnpredictableID = getMDKindID("unpredictable"); + assert(UnpredictableID == MD_unpredictable && + "unpredictable kind id drifted"); + (void)UnpredictableID; } LLVMContext::~LLVMContext() { delete pImpl; } Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -31,7 +31,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/InstCombine/InstCombine.h" #include "llvm/Transforms/Scalar/EarlyCSE.h" -#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" +#include "llvm/Transforms/Scalar/LowerPredictionIntrinsic.h" #include "llvm/Transforms/Scalar/SimplifyCFG.h" using namespace llvm; Index: lib/Passes/PassRegistry.def =================================================================== --- lib/Passes/PassRegistry.def +++ lib/Passes/PassRegistry.def @@ -67,7 +67,7 @@ FUNCTION_PASS("instcombine", InstCombinePass()) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("no-op-function", NoOpFunctionPass()) -FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass()) +FUNCTION_PASS("lower-prediction", LowerPredictionIntrinsicPass()) FUNCTION_PASS("print", PrintFunctionPass(dbgs())) FUNCTION_PASS("print", AssumptionPrinterPass(dbgs())) FUNCTION_PASS("print", DominatorTreePrinterPass(dbgs())) Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -177,7 +177,7 @@ else FPM.add(createScalarReplAggregatesPass()); FPM.add(createEarlyCSEPass()); - FPM.add(createLowerExpectIntrinsicPass()); + FPM.add(createLowerPredictionIntrinsicPass()); } void PassManagerBuilder::populateModulePassManager( Index: lib/Transforms/Scalar/CMakeLists.txt =================================================================== --- lib/Transforms/Scalar/CMakeLists.txt +++ lib/Transforms/Scalar/CMakeLists.txt @@ -27,7 +27,7 @@ LoopUnrollPass.cpp LoopUnswitch.cpp LowerAtomic.cpp - LowerExpectIntrinsic.cpp + LowerPredictionIntrinsic.cpp MemCpyOptimizer.cpp MergedLoadStoreMotion.cpp NaryReassociate.cpp Index: lib/Transforms/Scalar/LowerExpectIntrinsic.cpp =================================================================== --- lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -1,192 +0,0 @@ -//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This pass lowers the 'expect' intrinsic to LLVM metadata. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Metadata.h" -#include "llvm/Pass.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Transforms/Scalar.h" - -using namespace llvm; - -#define DEBUG_TYPE "lower-expect-intrinsic" - -STATISTIC(ExpectIntrinsicsHandled, - "Number of 'expect' intrinsic instructions handled"); - -static cl::opt -LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), - cl::desc("Weight of the branch likely to be taken (default = 64)")); -static cl::opt -UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), - cl::desc("Weight of the branch unlikely to be taken (default = 4)")); - -static bool handleSwitchExpect(SwitchInst &SI) { - CallInst *CI = dyn_cast(SI.getCondition()); - if (!CI) - return false; - - Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) - return false; - - Value *ArgValue = CI->getArgOperand(0); - ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); - if (!ExpectedValue) - return false; - - SwitchInst::CaseIt Case = SI.findCaseValue(ExpectedValue); - unsigned n = SI.getNumCases(); // +1 for default case. - SmallVector Weights(n + 1, UnlikelyBranchWeight); - - if (Case == SI.case_default()) - Weights[0] = LikelyBranchWeight; - else - Weights[Case.getCaseIndex() + 1] = LikelyBranchWeight; - - SI.setMetadata(LLVMContext::MD_prof, - MDBuilder(CI->getContext()).createBranchWeights(Weights)); - - SI.setCondition(ArgValue); - return true; -} - -static bool handleBranchExpect(BranchInst &BI) { - if (BI.isUnconditional()) - return false; - - // Handle non-optimized IR code like: - // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) - // %tobool = icmp ne i64 %expval, 0 - // br i1 %tobool, label %if.then, label %if.end - // - // Or the following simpler case: - // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) - // br i1 %expval, label %if.then, label %if.end - - CallInst *CI; - - ICmpInst *CmpI = dyn_cast(BI.getCondition()); - if (!CmpI) { - CI = dyn_cast(BI.getCondition()); - } else { - if (CmpI->getPredicate() != CmpInst::ICMP_NE) - return false; - CI = dyn_cast(CmpI->getOperand(0)); - } - - if (!CI) - return false; - - Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) - return false; - - Value *ArgValue = CI->getArgOperand(0); - ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); - if (!ExpectedValue) - return false; - - MDBuilder MDB(CI->getContext()); - MDNode *Node; - - // If expect value is equal to 1 it means that we are more likely to take - // branch 0, in other case more likely is branch 1. - if (ExpectedValue->isOne()) - Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); - else - Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); - - BI.setMetadata(LLVMContext::MD_prof, Node); - - if (CmpI) - CmpI->setOperand(0, ArgValue); - else - BI.setCondition(ArgValue); - return true; -} - -static bool lowerExpectIntrinsic(Function &F) { - bool Changed = false; - - for (BasicBlock &BB : F) { - // Create "block_weights" metadata. - if (BranchInst *BI = dyn_cast(BB.getTerminator())) { - if (handleBranchExpect(*BI)) - ExpectIntrinsicsHandled++; - } else if (SwitchInst *SI = dyn_cast(BB.getTerminator())) { - if (handleSwitchExpect(*SI)) - ExpectIntrinsicsHandled++; - } - - // Remove llvm.expect intrinsics. - for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) { - CallInst *CI = dyn_cast(BI++); - if (!CI) - continue; - - Function *Fn = CI->getCalledFunction(); - if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { - Value *Exp = CI->getArgOperand(0); - CI->replaceAllUsesWith(Exp); - CI->eraseFromParent(); - Changed = true; - } - } - } - - return Changed; -} - -PreservedAnalyses LowerExpectIntrinsicPass::run(Function &F) { - if (lowerExpectIntrinsic(F)) - return PreservedAnalyses::none(); - - return PreservedAnalyses::all(); -} - -namespace { -/// \brief Legacy pass for lowering expect intrinsics out of the IR. -/// -/// When this pass is run over a function it uses expect intrinsics which feed -/// branches and switches to provide branch weight metadata for those -/// terminators. It then removes the expect intrinsics from the IR so the rest -/// of the optimizer can ignore them. -class LowerExpectIntrinsic : public FunctionPass { -public: - static char ID; - LowerExpectIntrinsic() : FunctionPass(ID) { - initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); } -}; -} - -char LowerExpectIntrinsic::ID = 0; -INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", - "Lower 'expect' Intrinsics", false, false) - -FunctionPass *llvm::createLowerExpectIntrinsicPass() { - return new LowerExpectIntrinsic(); -} Index: lib/Transforms/Scalar/LowerPredictionIntrinsic.cpp =================================================================== --- lib/Transforms/Scalar/LowerPredictionIntrinsic.cpp +++ lib/Transforms/Scalar/LowerPredictionIntrinsic.cpp @@ -1,4 +1,4 @@ -//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// +//===- LowerPredictionIntrinsic.cpp - Lower prediction intrinsics ---------===// // // The LLVM Compiler Infrastructure // @@ -7,11 +7,11 @@ // //===----------------------------------------------------------------------===// // -// This pass lowers the 'expect' intrinsic to LLVM metadata. +// This pass lowers 'expect' or 'unpredictable' intrinsics to LLVM metadata. // //===----------------------------------------------------------------------===// -#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" +#include "llvm/Transforms/Scalar/LowerPredictionIntrinsic.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/BasicBlock.h" @@ -31,8 +31,8 @@ #define DEBUG_TYPE "lower-expect-intrinsic" -STATISTIC(ExpectIntrinsicsHandled, - "Number of 'expect' intrinsic instructions handled"); +STATISTIC(BranchIntrinsicsHandled, + "Number of 'expect' or 'unpredictable' intrinsics handled"); static cl::opt LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), @@ -41,6 +41,7 @@ UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), cl::desc("Weight of the branch unlikely to be taken (default = 4)")); +// TODO: Handle unpredictable for a switch too. static bool handleSwitchExpect(SwitchInst &SI) { CallInst *CI = dyn_cast(SI.getCondition()); if (!CI) @@ -71,7 +72,27 @@ return true; } -static bool handleBranchExpect(BranchInst &BI) { +static void setMetadataForExpect(BranchInst &BI, CallInst &CI, + ConstantInt &ExpectedValue) { + MDBuilder MDB(CI.getContext()); + MDNode *Node; + + // If expect value is equal to 1 it means that we are more likely to take + // branch 0, in other case more likely is branch 1. + if (ExpectedValue.isOne()) + Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); + else + Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); + + BI.setMetadata(LLVMContext::MD_prof, Node); +} + +static void setMetadataForUnpredictable(BranchInst &BI, CallInst &CI) { + MDNode *MD = MDNode::get(CI.getContext(), None); + BI.setMetadata(LLVMContext::MD_unpredictable, MD); +} + +static bool handleBranchIntrinsic(BranchInst &BI) { if (BI.isUnconditional()) return false; @@ -99,26 +120,20 @@ return false; Function *Fn = CI->getCalledFunction(); - if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) + if (!Fn) return false; - Value *ArgValue = CI->getArgOperand(0); - ConstantInt *ExpectedValue = dyn_cast(CI->getArgOperand(1)); - if (!ExpectedValue) + if (Fn->getIntrinsicID() == Intrinsic::expect) { + if (auto *ExpectedValue = dyn_cast(CI->getArgOperand(1))) + setMetadataForExpect(BI, *CI, *ExpectedValue); + else + return false; + } else if (Fn->getIntrinsicID() == Intrinsic::unpredictable) { + setMetadataForUnpredictable(BI, *CI); + } else return false; - MDBuilder MDB(CI->getContext()); - MDNode *Node; - - // If expect value is equal to 1 it means that we are more likely to take - // branch 0, in other case more likely is branch 1. - if (ExpectedValue->isOne()) - Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); - else - Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); - - BI.setMetadata(LLVMContext::MD_prof, Node); - + Value *ArgValue = CI->getArgOperand(0); if (CmpI) CmpI->setOperand(0, ArgValue); else @@ -126,27 +141,31 @@ return true; } -static bool lowerExpectIntrinsic(Function &F) { +static bool lowerPredictionIntrinsics(Function &F) { bool Changed = false; for (BasicBlock &BB : F) { - // Create "block_weights" metadata. + // Create metadata. if (BranchInst *BI = dyn_cast(BB.getTerminator())) { - if (handleBranchExpect(*BI)) - ExpectIntrinsicsHandled++; + if (handleBranchIntrinsic(*BI)) + BranchIntrinsicsHandled++; } else if (SwitchInst *SI = dyn_cast(BB.getTerminator())) { if (handleSwitchExpect(*SI)) - ExpectIntrinsicsHandled++; + BranchIntrinsicsHandled++; } - // Remove llvm.expect intrinsics. + // Remove intrinsics. for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) { CallInst *CI = dyn_cast(BI++); if (!CI) continue; Function *Fn = CI->getCalledFunction(); - if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { + if (!Fn) + continue; + + if (Fn->getIntrinsicID() == Intrinsic::expect || + Fn->getIntrinsicID() == Intrinsic::unpredictable) { Value *Exp = CI->getArgOperand(0); CI->replaceAllUsesWith(Exp); CI->eraseFromParent(); @@ -158,35 +177,38 @@ return Changed; } -PreservedAnalyses LowerExpectIntrinsicPass::run(Function &F) { - if (lowerExpectIntrinsic(F)) +PreservedAnalyses LowerPredictionIntrinsicPass::run(Function &F) { + if (lowerPredictionIntrinsics(F)) return PreservedAnalyses::none(); return PreservedAnalyses::all(); } namespace { -/// \brief Legacy pass for lowering expect intrinsics out of the IR. +/// \brief Legacy pass for lowering expect or unpredictable intrinsics out of +/// the IR. /// -/// When this pass is run over a function it uses expect intrinsics which feed -/// branches and switches to provide branch weight metadata for those -/// terminators. It then removes the expect intrinsics from the IR so the rest -/// of the optimizer can ignore them. -class LowerExpectIntrinsic : public FunctionPass { +/// When this pass is run over a function it uses expect or unpredictable +/// intrinsics which feed branches and switches to provide branch weight or +/// unpredictable metadata for those terminators. It then removes the intrinsics +/// from the IR so the rest of the optimizer can ignore them. +class LowerPredictionIntrinsic : public FunctionPass { public: static char ID; - LowerExpectIntrinsic() : FunctionPass(ID) { - initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); + LowerPredictionIntrinsic() : FunctionPass(ID) { + initializeLowerPredictionIntrinsicPass(*PassRegistry::getPassRegistry()); } - bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); } + bool runOnFunction(Function &F) override { + return lowerPredictionIntrinsics(F); + } }; } -char LowerExpectIntrinsic::ID = 0; -INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", - "Lower 'expect' Intrinsics", false, false) +char LowerPredictionIntrinsic::ID = 0; +INITIALIZE_PASS(LowerPredictionIntrinsic, "lower-prediction", + "Lower 'expect' or 'unpredictable' intrinsics", false, false) -FunctionPass *llvm::createLowerExpectIntrinsicPass() { - return new LowerExpectIntrinsic(); +FunctionPass *llvm::createLowerPredictionIntrinsicPass() { + return new LowerPredictionIntrinsic(); } Index: lib/Transforms/Scalar/Scalar.cpp =================================================================== --- lib/Transforms/Scalar/Scalar.cpp +++ lib/Transforms/Scalar/Scalar.cpp @@ -58,7 +58,7 @@ initializeLoopUnswitchPass(Registry); initializeLoopIdiomRecognizePass(Registry); initializeLowerAtomicPass(Registry); - initializeLowerExpectIntrinsicPass(Registry); + initializeLowerPredictionIntrinsicPass(Registry); initializeMemCpyOptPass(Registry); initializeMergedLoadStoreMotionPass(Registry); initializeNaryReassociatePass(Registry); @@ -238,6 +238,6 @@ unwrap(PM)->add(createBasicAliasAnalysisPass()); } -void LLVMAddLowerExpectIntrinsicPass(LLVMPassManagerRef PM) { - unwrap(PM)->add(createLowerExpectIntrinsicPass()); +void LLVMAddLowerPredictionIntrinsicPass(LLVMPassManagerRef PM) { + unwrap(PM)->add(createLowerPredictionIntrinsicPass()); } Index: test/Transforms/LowerExpectIntrinsic/basic.ll =================================================================== --- test/Transforms/LowerExpectIntrinsic/basic.ll +++ test/Transforms/LowerExpectIntrinsic/basic.ll @@ -1,5 +1,5 @@ -; RUN: opt -lower-expect -strip-dead-prototypes -S -o - < %s | FileCheck %s -; RUN: opt -S -passes=lower-expect < %s | opt -strip-dead-prototypes -S | FileCheck %s +; RUN: opt -lower-prediction -strip-dead-prototypes -S -o - < %s | FileCheck %s +; RUN: opt -S -passes=lower-prediction < %s | opt -strip-dead-prototypes -S | FileCheck %s ; CHECK-LABEL: @test1( define i32 @test1(i32 %x) nounwind uwtable ssp { Index: test/Transforms/LowerExpectIntrinsic/unpredictable.ll =================================================================== --- test/Transforms/LowerExpectIntrinsic/unpredictable.ll +++ test/Transforms/LowerExpectIntrinsic/unpredictable.ll @@ -0,0 +1,285 @@ +; RUN: opt -lower-prediction -strip-dead-prototypes -S -o - < %s | FileCheck %s +; RUN: opt -S -passes=lower-prediction < %s | opt -strip-dead-prototypes -S | FileCheck %s + +; These tests are identical to the llvm.expect tests except that all of the +; llvm.expect intrinsic calls are replaced by llvm.unpredictable. + +; CHECK-LABEL: @test1( +define i32 @test1(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %cmp = icmp sgt i32 %tmp, 1 + %conv = zext i1 %cmp to i32 + %conv1 = sext i32 %conv to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv1) + %tobool = icmp ne i64 %unpredictable, 0 +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +declare i64 @llvm.unpredictable.i64(i64) nounwind readnone + +declare i32 @f(...) + +; CHECK-LABEL: @test2( +define i32 @test2(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %conv = sext i32 %tmp to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv) + %tobool = icmp ne i64 %unpredictable, 0 +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +; CHECK-LABEL: @test3( +define i32 @test3(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %tobool = icmp ne i32 %tmp, 0 + %lnot = xor i1 %tobool, true + %lnot.ext = zext i1 %lnot to i32 + %conv = sext i32 %lnot.ext to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv) + %tobool1 = icmp ne i64 %unpredictable, 0 +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %tobool1, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +; CHECK-LABEL: @test4( +define i32 @test4(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %tobool = icmp ne i32 %tmp, 0 + %lnot = xor i1 %tobool, true + %lnot1 = xor i1 %lnot, true + %lnot.ext = zext i1 %lnot1 to i32 + %conv = sext i32 %lnot.ext to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv) + %tobool2 = icmp ne i64 %unpredictable, 0 +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %tobool2, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +; CHECK-LABEL: @test5( +define i32 @test5(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %cmp = icmp slt i32 %tmp, 0 + %conv = zext i1 %cmp to i32 + %conv1 = sext i32 %conv to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv1) + %tobool = icmp ne i64 %unpredictable, 0 +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +; FIXME: Unpredictable switches are not implemented yet. + + +; CHECK-LABEL: @test6( +define i32 @test6(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %conv = sext i32 %tmp to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv) +; FIXME-CHECK: !unpredictable !0 +; FIXME-CHECK-NOT: @llvm.unpredictable + switch i64 %unpredictable, label %sw.epilog [ + i64 1, label %sw.bb + i64 2, label %sw.bb + ] + +sw.bb: ; preds = %entry, %entry + store i32 0, i32* %retval + br label %return + +sw.epilog: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %sw.epilog, %sw.bb + %0 = load i32, i32* %retval + ret i32 %0 +} + +; CHECK-LABEL: @test7( +define i32 @test7(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %conv = sext i32 %tmp to i64 + %unpredictable = call i64 @llvm.unpredictable.i64(i64 %conv) +; FIXME-CHECK: !unpredictable !0 +; FIXME-CHECK-NOT: @llvm.unpredictable + switch i64 %unpredictable, label %sw.epilog [ + i64 2, label %sw.bb + i64 3, label %sw.bb + ] + +sw.bb: ; preds = %entry, %entry + %tmp1 = load i32, i32* %x.addr, align 4 + store i32 %tmp1, i32* %retval + br label %return + +sw.epilog: ; preds = %entry + store i32 0, i32* %retval + br label %return + +return: ; preds = %sw.epilog, %sw.bb + %0 = load i32, i32* %retval + ret i32 %0 +} + +; CHECK-LABEL: @test8( +define i32 @test8(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %cmp = icmp sgt i32 %tmp, 1 + %conv = zext i1 %cmp to i32 + %unpredictable = call i32 @llvm.unpredictable.i32(i32 %conv) + %tobool = icmp ne i32 %unpredictable, 0 +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +declare i32 @llvm.unpredictable.i32(i32) nounwind readnone + +; CHECK-LABEL: @test9( +define i32 @test9(i32 %x) nounwind uwtable ssp { +entry: + %retval = alloca i32, align 4 + %x.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + %tmp = load i32, i32* %x.addr, align 4 + %cmp = icmp sgt i32 %tmp, 1 + %unpredictable = call i1 @llvm.unpredictable.i1(i1 %cmp) +; CHECK: !unpredictable !0 +; CHECK-NOT: @llvm.unpredictable + br i1 %unpredictable, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 (...) @f() + store i32 %call, i32* %retval + br label %return + +if.end: ; preds = %entry + store i32 1, i32* %retval + br label %return + +return: ; preds = %if.end, %if.then + %0 = load i32, i32* %retval + ret i32 %0 +} + +declare i1 @llvm.unpredictable.i1(i1) nounwind readnone + +; CHECK: !0 = !{} +