Index: llvm/include/llvm/IR/AbstractCallSite.h =================================================================== --- /dev/null +++ llvm/include/llvm/IR/AbstractCallSite.h @@ -0,0 +1,205 @@ +//===- AbstractCallSite.h - Abstract call site ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// An abstract call site is a wrapper that allows to treat direct, +// indirect, and callback calls the same. If an abstract call site +// represents a direct or indirect call site it behaves like a stripped +// down version of a normal call site object. The abstract call site can +// also represent a callback call, thus the fact that the initially +// called function (=broker) may invoke a third one (=callback callee). +// In this case, the abstract call site hides the middle man, hence the +// broker function. The result is a representation of the callback call, +// inside the broker, but in the context of the original call to the broker. +// +// There are up to three functions involved when we talk about callback call +// sites. The caller (1), which invokes the broker function. The broker +// function (2), that will invoke the callee zero or more times. And finally +// the callee (3), which is the target of the callback call. +// +// The abstract call site will handle the mapping from parameters to arguments +// depending on the semantic of the broker function. However, it is important +// to note that the mapping is often partial. Thus, some arguments of the +// call/invoke instruction are mapped to parameters of the callee while others +// are not. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_ABSTRACTCALLSITE_H +#define LLVM_IR_ABSTRACTCALLSITE_H + +#include "llvm/IR/InstrTypes.h" + +namespace llvm { + +class AbstractCallSite { +public: + /// The encoding of a callback with regards to the underlying instruction. + struct CallbackInfo { + + /// For direct/indirect calls the parameter encoding is empty. If it is not, + /// the abstract call site represents a callback. In that case, the first + /// element of the encoding vector represents which argument of the call + /// site CB is the callback callee. The remaining elements map parameters + /// (identified by their position) to the arguments that will be passed + /// through (also identified by position but in the call site instruction). + /// + /// NOTE that we use LLVM argument numbers (starting at 0) and not + /// clang/source argument numbers (starting at 1). The -1 entries represent + /// unknown values that are passed to the callee. + using ParameterEncodingTy = SmallVector; + ParameterEncodingTy ParameterEncoding; + }; + +private: + /// The underlying call site: + /// caller -> callee, if this is a direct or indirect call site + /// caller -> broker function, if this is a callback call site + CallBase *CB; + + /// The encoding of a callback with regards to the underlying instruction. + CallbackInfo CI; + +public: + /// Sole constructor for abstract call sites (ACS). + /// + /// An abstract call site can only be constructed through a llvm::Use because + /// each operand (=use) of an instruction could potentially be a different + /// abstract call site. Furthermore, even if the value of the llvm::Use is the + /// same, and the user is as well, the abstract call sites might not be. + /// + /// If a use is not associated with an abstract call site the constructed ACS + /// will evaluate to false if converted to a boolean. + /// + /// If the use is the callee use of a call or invoke instruction, the + /// constructed abstract call site will behave as a llvm::CallSite would. + /// + /// If the use is not a callee use of a call or invoke instruction, the + /// callback metadata is used to determine the argument <-> parameter mapping + /// as well as the callee of the abstract call site. + AbstractCallSite(const Use *U); + + /// Add operand uses of \p CB that represent callback uses into \p CBUses. + /// + /// All uses added to \p CallbackUses can be used to create abstract call + /// sites for which AbstractCallSite::isCallbackCall() will return true. + static void getCallbackUses(const CallBase &CB, + SmallVectorImpl &CallbackUses); + + /// Conversion operator to conveniently check for a valid/initialized ACS. + explicit operator bool() const { return CB != nullptr; } + + /// Return the underlying instruction. + CallBase *getInstruction() const { return CB; } + + /// Return true if this ACS represents a direct call. + bool isDirectCall() const { + return !isCallbackCall() && !CB->isIndirectCall(); + } + + /// Return true if this ACS represents an indirect call. + bool isIndirectCall() const { + return !isCallbackCall() && CB->isIndirectCall(); + } + + /// Return true if this ACS represents a callback call. + bool isCallbackCall() const { + // For a callback call site the callee is ALWAYS stored first in the + // transitive values vector. Thus, a non-empty vector indicates a callback. + return !CI.ParameterEncoding.empty(); + } + + /// Return true if @p UI is the use that defines the callee of this ACS. + bool isCallee(Value::const_user_iterator UI) const { + return isCallee(&UI.getUse()); + } + + /// Return true if @p U is the use that defines the callee of this ACS. + bool isCallee(const Use *U) const { + if (isDirectCall()) + return CB->isCallee(U); + + assert(!CI.ParameterEncoding.empty() && + "Callback without parameter encoding!"); + + return (int)CB->getArgOperandNo(U) == CI.ParameterEncoding[0]; + } + + /// Return the number of parameters of the callee. + unsigned getNumArgOperands() const { + if (isDirectCall()) + return CB->getNumArgOperands(); + // Subtract 1 for the callee encoding. + return CI.ParameterEncoding.size() - 1; + } + + /// Return the operand index of the underlying instruction associated with @p + /// Arg. + int getCallArgOperandNo(Argument &Arg) const { + return getCallArgOperandNo(Arg.getArgNo()); + } + + /// Return the operand index of the underlying instruction associated with + /// the function parameter number @p ArgNo or -1 if there is none. + int getCallArgOperandNo(unsigned ArgNo) const { + if (isDirectCall()) + return ArgNo; + // Add 1 for the callee encoding. + return CI.ParameterEncoding[ArgNo + 1]; + } + + /// Return the operand of the underlying instruction associated with @p Arg. + Value *getCallArgOperand(Argument &Arg) const { + return getCallArgOperand(Arg.getArgNo()); + } + + /// Return the operand of the underlying instruction associated with the + /// function parameter number @p ArgNo or nullptr if there is none. + Value *getCallArgOperand(unsigned ArgNo) const { + if (isDirectCall()) + return CB->getArgOperand(ArgNo); + // Add 1 for the callee encoding. + return CI.ParameterEncoding[ArgNo + 1] >= 0 + ? CB->getArgOperand(CI.ParameterEncoding[ArgNo + 1]) + : nullptr; + } + + /// Return the operand index of the underlying instruction associated with the + /// callee of this ACS. Only valid for callback calls! + int getCallArgOperandNoForCallee() const { + assert(isCallbackCall()); + assert(CI.ParameterEncoding.size() && CI.ParameterEncoding[0] >= 0); + return CI.ParameterEncoding[0]; + } + + /// Return the use of the callee value in the underlying instruction. Only + /// valid for callback calls! + const Use &getCalleeUseForCallback() const { + int CalleeArgIdx = getCallArgOperandNoForCallee(); + assert(CalleeArgIdx >= 0 && + unsigned(CalleeArgIdx) < getInstruction()->getNumOperands()); + return getInstruction()->getOperandUse(CalleeArgIdx); + } + + /// Return the pointer to function that is being called. + Value *getCalledValue() const { + if (isDirectCall()) + return CB->getCalledValue(); + return CB->getArgOperand(getCallArgOperandNoForCallee()); + } + + /// Return the function being called if this is a direct call, otherwise + /// return null (if it's an indirect call). + Function *getCalledFunction() const { + Value *V = getCalledValue(); + return V ? dyn_cast(V->stripPointerCasts()) : nullptr; + } +}; + +} // end namespace llvm + +#endif // LLVM_IR_ABSTRACTCALLSITE_H Index: llvm/include/llvm/IR/CallSite.h =================================================================== --- llvm/include/llvm/IR/CallSite.h +++ llvm/include/llvm/IR/CallSite.h @@ -726,198 +726,6 @@ ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {} }; -/// AbstractCallSite -/// -/// An abstract call site is a wrapper that allows to treat direct, -/// indirect, and callback calls the same. If an abstract call site -/// represents a direct or indirect call site it behaves like a stripped -/// down version of a normal call site object. The abstract call site can -/// also represent a callback call, thus the fact that the initially -/// called function (=broker) may invoke a third one (=callback callee). -/// In this case, the abstract call site hides the middle man, hence the -/// broker function. The result is a representation of the callback call, -/// inside the broker, but in the context of the original call to the broker. -/// -/// There are up to three functions involved when we talk about callback call -/// sites. The caller (1), which invokes the broker function. The broker -/// function (2), that will invoke the callee zero or more times. And finally -/// the callee (3), which is the target of the callback call. -/// -/// The abstract call site will handle the mapping from parameters to arguments -/// depending on the semantic of the broker function. However, it is important -/// to note that the mapping is often partial. Thus, some arguments of the -/// call/invoke instruction are mapped to parameters of the callee while others -/// are not. -class AbstractCallSite { -public: - - /// The encoding of a callback with regards to the underlying instruction. - struct CallbackInfo { - - /// For direct/indirect calls the parameter encoding is empty. If it is not, - /// the abstract call site represents a callback. In that case, the first - /// element of the encoding vector represents which argument of the call - /// site CS is the callback callee. The remaining elements map parameters - /// (identified by their position) to the arguments that will be passed - /// through (also identified by position but in the call site instruction). - /// - /// NOTE that we use LLVM argument numbers (starting at 0) and not - /// clang/source argument numbers (starting at 1). The -1 entries represent - /// unknown values that are passed to the callee. - using ParameterEncodingTy = SmallVector; - ParameterEncodingTy ParameterEncoding; - - }; - -private: - - /// The underlying call site: - /// caller -> callee, if this is a direct or indirect call site - /// caller -> broker function, if this is a callback call site - CallSite CS; - - /// The encoding of a callback with regards to the underlying instruction. - CallbackInfo CI; - -public: - /// Sole constructor for abstract call sites (ACS). - /// - /// An abstract call site can only be constructed through a llvm::Use because - /// each operand (=use) of an instruction could potentially be a different - /// abstract call site. Furthermore, even if the value of the llvm::Use is the - /// same, and the user is as well, the abstract call sites might not be. - /// - /// If a use is not associated with an abstract call site the constructed ACS - /// will evaluate to false if converted to a boolean. - /// - /// If the use is the callee use of a call or invoke instruction, the - /// constructed abstract call site will behave as a llvm::CallSite would. - /// - /// If the use is not a callee use of a call or invoke instruction, the - /// callback metadata is used to determine the argument <-> parameter mapping - /// as well as the callee of the abstract call site. - AbstractCallSite(const Use *U); - - /// Add operand uses of \p ICS that represent callback uses into \p CBUses. - /// - /// All uses added to \p CBUses can be used to create abstract call sites for - /// which AbstractCallSite::isCallbackCall() will return true. - static void getCallbackUses(ImmutableCallSite ICS, - SmallVectorImpl &CBUses); - - /// Conversion operator to conveniently check for a valid/initialized ACS. - explicit operator bool() const { return (bool)CS; } - - /// Return the underlying instruction. - Instruction *getInstruction() const { return CS.getInstruction(); } - - /// Return the call site abstraction for the underlying instruction. - CallSite getCallSite() const { return CS; } - - /// Return true if this ACS represents a direct call. - bool isDirectCall() const { - return !isCallbackCall() && !CS.isIndirectCall(); - } - - /// Return true if this ACS represents an indirect call. - bool isIndirectCall() const { - return !isCallbackCall() && CS.isIndirectCall(); - } - - /// Return true if this ACS represents a callback call. - bool isCallbackCall() const { - // For a callback call site the callee is ALWAYS stored first in the - // transitive values vector. Thus, a non-empty vector indicates a callback. - return !CI.ParameterEncoding.empty(); - } - - /// Return true if @p UI is the use that defines the callee of this ACS. - bool isCallee(Value::const_user_iterator UI) const { - return isCallee(&UI.getUse()); - } - - /// Return true if @p U is the use that defines the callee of this ACS. - bool isCallee(const Use *U) const { - if (isDirectCall()) - return CS.isCallee(U); - - assert(!CI.ParameterEncoding.empty() && - "Callback without parameter encoding!"); - - return (int)CS.getArgumentNo(U) == CI.ParameterEncoding[0]; - } - - /// Return the number of parameters of the callee. - unsigned getNumArgOperands() const { - if (isDirectCall()) - return CS.getNumArgOperands(); - // Subtract 1 for the callee encoding. - return CI.ParameterEncoding.size() - 1; - } - - /// Return the operand index of the underlying instruction associated with @p - /// Arg. - int getCallArgOperandNo(Argument &Arg) const { - return getCallArgOperandNo(Arg.getArgNo()); - } - - /// Return the operand index of the underlying instruction associated with - /// the function parameter number @p ArgNo or -1 if there is none. - int getCallArgOperandNo(unsigned ArgNo) const { - if (isDirectCall()) - return ArgNo; - // Add 1 for the callee encoding. - return CI.ParameterEncoding[ArgNo + 1]; - } - - /// Return the operand of the underlying instruction associated with @p Arg. - Value *getCallArgOperand(Argument &Arg) const { - return getCallArgOperand(Arg.getArgNo()); - } - - /// Return the operand of the underlying instruction associated with the - /// function parameter number @p ArgNo or nullptr if there is none. - Value *getCallArgOperand(unsigned ArgNo) const { - if (isDirectCall()) - return CS.getArgOperand(ArgNo); - // Add 1 for the callee encoding. - return CI.ParameterEncoding[ArgNo + 1] >= 0 - ? CS.getArgOperand(CI.ParameterEncoding[ArgNo + 1]) - : nullptr; - } - - /// Return the operand index of the underlying instruction associated with the - /// callee of this ACS. Only valid for callback calls! - int getCallArgOperandNoForCallee() const { - assert(isCallbackCall()); - assert(CI.ParameterEncoding.size() && CI.ParameterEncoding[0] >= 0); - return CI.ParameterEncoding[0]; - } - - /// Return the use of the callee value in the underlying instruction. Only - /// valid for callback calls! - const Use &getCalleeUseForCallback() const { - int CalleeArgIdx = getCallArgOperandNoForCallee(); - assert(CalleeArgIdx >= 0 && - unsigned(CalleeArgIdx) < getInstruction()->getNumOperands()); - return getInstruction()->getOperandUse(CalleeArgIdx); - } - - /// Return the pointer to function that is being called. - Value *getCalledValue() const { - if (isDirectCall()) - return CS.getCalledValue(); - return CS.getArgOperand(getCallArgOperandNoForCallee()); - } - - /// Return the function being called if this is a direct call, otherwise - /// return null (if it's an indirect call). - Function *getCalledFunction() const { - Value *V = getCalledValue(); - return V ? dyn_cast(V->stripPointerCasts()) : nullptr; - } -}; - template <> struct DenseMapInfo { using BaseInfo = DenseMapInfo; Index: llvm/include/llvm/Transforms/IPO/Attributor.h =================================================================== --- llvm/include/llvm/Transforms/IPO/Attributor.h +++ llvm/include/llvm/Transforms/IPO/Attributor.h @@ -111,6 +111,7 @@ #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/IR/AbstractCallSite.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/PassManager.h" Index: llvm/lib/IR/AbstractCallSite.cpp =================================================================== --- llvm/lib/IR/AbstractCallSite.cpp +++ llvm/lib/IR/AbstractCallSite.cpp @@ -14,9 +14,9 @@ // //===----------------------------------------------------------------------===// +#include "llvm/IR/AbstractCallSite.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringSwitch.h" -#include "llvm/IR/CallSite.h" #include "llvm/Support/Debug.h" using namespace llvm; @@ -33,9 +33,9 @@ STATISTIC(NumInvalidAbstractCallSitesNoCallback, "Number of invalid abstract call sites created (no callback)"); -void AbstractCallSite::getCallbackUses(ImmutableCallSite ICS, - SmallVectorImpl &CBUses) { - const Function *Callee = ICS.getCalledFunction(); +void AbstractCallSite::getCallbackUses( + const CallBase &CB, SmallVectorImpl &CallbackUses) { + const Function *Callee = CB.getCalledFunction(); if (!Callee) return; @@ -45,60 +45,61 @@ for (const MDOperand &Op : CallbackMD->operands()) { MDNode *OpMD = cast(Op.get()); - auto *CBCalleeIdxAsCM = cast(OpMD->getOperand(0)); - uint64_t CBCalleeIdx = - cast(CBCalleeIdxAsCM->getValue())->getZExtValue(); - if (CBCalleeIdx < ICS.arg_size()) - CBUses.push_back(ICS.arg_begin() + CBCalleeIdx); + auto *CallbackCalleeIdxAsCM = cast(OpMD->getOperand(0)); + uint64_t CallbackCalleeIdx = + cast(CallbackCalleeIdxAsCM->getValue())->getZExtValue(); + if (CallbackCalleeIdx < CB.arg_size()) + CallbackUses.push_back(CB.arg_begin() + CallbackCalleeIdx); } } /// Create an abstract call site from a use. -AbstractCallSite::AbstractCallSite(const Use *U) : CS(U->getUser()) { +AbstractCallSite::AbstractCallSite(const Use *U) + : CB(dyn_cast(U->getUser())) { // First handle unknown users. - if (!CS) { + if (!CB) { // If the use is actually in a constant cast expression which itself // has only one use, we look through the constant cast expression. // This happens by updating the use @p U to the use of the constant - // cast expression and afterwards re-initializing CS accordingly. + // cast expression and afterwards re-initializing CB accordingly. if (ConstantExpr *CE = dyn_cast(U->getUser())) if (CE->getNumUses() == 1 && CE->isCast()) { U = &*CE->use_begin(); - CS = CallSite(U->getUser()); + CB = dyn_cast(U->getUser()); } - if (!CS) { + if (!CB) { NumInvalidAbstractCallSitesUnknownUse++; return; } } // Then handle direct or indirect calls. Thus, if U is the callee of the - // call site CS it is not a callback and we are done. - if (CS.isCallee(U)) { + // call site CB it is not a callback and we are done. + if (CB->isCallee(U)) { NumDirectAbstractCallSites++; return; } // If we cannot identify the broker function we cannot create a callback and // invalidate the abstract call site. - Function *Callee = CS.getCalledFunction(); + Function *Callee = CB->getCalledFunction(); if (!Callee) { NumInvalidAbstractCallSitesUnknownCallee++; - CS = CallSite(); + CB = nullptr; return; } MDNode *CallbackMD = Callee->getMetadata(LLVMContext::MD_callback); if (!CallbackMD) { NumInvalidAbstractCallSitesNoCallback++; - CS = CallSite(); + CB = nullptr; return; } - unsigned UseIdx = CS.getArgumentNo(U); + unsigned UseIdx = CB->getArgOperandNo(U); MDNode *CallbackEncMD = nullptr; for (const MDOperand &Op : CallbackMD->operands()) { MDNode *OpMD = cast(Op.get()); @@ -113,7 +114,7 @@ if (!CallbackEncMD) { NumInvalidAbstractCallSitesNoCallback++; - CS = CallSite(); + CB = nullptr; return; } @@ -121,7 +122,7 @@ assert(CallbackEncMD->getNumOperands() >= 2 && "Incomplete !callback metadata"); - unsigned NumCallOperands = CS.getNumArgOperands(); + unsigned NumCallOperands = CB->getNumArgOperands(); // Skip the var-arg flag at the end when reading the metadata. for (unsigned u = 0, e = CallbackEncMD->getNumOperands() - 1; u < e; u++) { Metadata *OpAsM = CallbackEncMD->getOperand(u).get(); Index: llvm/lib/Transforms/IPO/Attributor.cpp =================================================================== --- llvm/lib/Transforms/IPO/Attributor.cpp +++ llvm/lib/Transforms/IPO/Attributor.cpp @@ -151,10 +151,10 @@ // of the underlying call site operand, we want the corresponding callback // callee argument and not the direct callee argument. Optional CBCandidateArg; - SmallVector CBUses; - ImmutableCallSite ICS(&getAnchorValue()); - AbstractCallSite::getCallbackUses(ICS, CBUses); - for (const Use *U : CBUses) { + SmallVector CallbackUses; + const auto &CB = cast(getAnchorValue()); + AbstractCallSite::getCallbackUses(CB, CallbackUses); + for (const Use *U : CallbackUses) { AbstractCallSite ACS(U); assert(ACS && ACS.isCallbackCall()); if (!ACS.getCalledFunction()) @@ -183,7 +183,7 @@ // If no callbacks were found, or none used the underlying call site operand // exclusively, use the direct callee argument if available. - const Function *Callee = ICS.getCalledFunction(); + const Function *Callee = CB.getCalledFunction(); if (Callee && Callee->arg_size() > unsigned(ArgNo)) return Callee->getArg(ArgNo); @@ -1328,7 +1328,7 @@ auto CallSiteCanBeChanged = [](AbstractCallSite ACS) { // Forbid must-tail calls for now. - return !ACS.isCallbackCall() && !ACS.getCallSite().isMustTailCall(); + return !ACS.isCallbackCall() && !ACS.getInstruction()->isMustTailCall(); }; Function *Fn = Arg.getParent(); Index: llvm/lib/Transforms/IPO/AttributorAttributes.cpp =================================================================== --- llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -4993,9 +4993,10 @@ // Helper to check if for the given call site the associated argument is // passed to a callback where the privatization would be different. auto IsCompatiblePrivArgOfCallback = [&](CallSite CS) { - SmallVector CBUses; - AbstractCallSite::getCallbackUses(CS, CBUses); - for (const Use *U : CBUses) { + SmallVector CallbackUses; + AbstractCallSite::getCallbackUses(cast(*CS.getInstruction()), + CallbackUses); + for (const Use *U : CallbackUses) { AbstractCallSite CBACS(U); assert(CBACS && CBACS.isCallbackCall()); for (Argument &CBArg : CBACS.getCalledFunction()->args()) { @@ -5081,7 +5082,7 @@ << Arg->getParent()->getName() << ")\n[AAPrivatizablePtr] because it is an argument in a " "direct call of (" - << ACS.getCallSite().getCalledFunction()->getName() + << ACS.getInstruction()->getCalledFunction()->getName() << ").\n[AAPrivatizablePtr] for which the argument " "privatization is not compatible.\n"; }); @@ -5093,7 +5094,7 @@ // here. auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) { if (ACS.isDirectCall()) - return IsCompatiblePrivArgOfCallback(ACS.getCallSite()); + return IsCompatiblePrivArgOfCallback(CallSite(ACS.getInstruction())); if (ACS.isCallbackCall()) return IsCompatiblePrivArgOfDirectCS(ACS); return false; Index: llvm/lib/Transforms/IPO/IPConstantPropagation.cpp =================================================================== --- llvm/lib/Transforms/IPO/IPConstantPropagation.cpp +++ llvm/lib/Transforms/IPO/IPConstantPropagation.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/AbstractCallSite.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h"