Index: llvm/trunk/include/llvm/Transforms/IPO/ArgumentPromotion.h =================================================================== --- llvm/trunk/include/llvm/Transforms/IPO/ArgumentPromotion.h +++ llvm/trunk/include/llvm/Transforms/IPO/ArgumentPromotion.h @@ -0,0 +1,31 @@ +//===- ArgumentPromotion.h - Promote by-reference arguments -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_ARGUMENTPROMOTION_H +#define LLVM_TRANSFORMS_IPO_ARGUMENTPROMOTION_H + +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/LazyCallGraph.h" + +namespace llvm { + +/// Argument promotion pass. +/// +/// This pass walks the functions in each SCC and for each one tries to +/// transform it and all of its callers to replace indirect arguments with +/// direct (by-value) arguments. +class ArgumentPromotionPass : public PassInfoMixin { +public: + PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, + LazyCallGraph &CG, CGSCCUpdateResult &UR); +}; + +} + +#endif Index: llvm/trunk/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/trunk/lib/Passes/PassBuilder.cpp +++ llvm/trunk/lib/Passes/PassBuilder.cpp @@ -61,6 +61,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/GCOVProfiler.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/Transforms/IPO/ConstantMerge.h" #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" Index: llvm/trunk/lib/Passes/PassRegistry.def =================================================================== --- llvm/trunk/lib/Passes/PassRegistry.def +++ llvm/trunk/lib/Passes/PassRegistry.def @@ -85,6 +85,7 @@ #ifndef CGSCC_PASS #define CGSCC_PASS(NAME, CREATE_PASS) #endif +CGSCC_PASS("argpromotion", ArgumentPromotionPass()) CGSCC_PASS("invalidate", InvalidateAllAnalysesPass()) CGSCC_PASS("function-attrs", PostOrderFunctionAttrsPass()) CGSCC_PASS("inline", InlinerPass()) Index: llvm/trunk/lib/Transforms/IPO/ArgumentPromotion.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/ArgumentPromotion.cpp +++ llvm/trunk/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -29,7 +29,9 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -37,6 +39,7 @@ #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CFG.h" @@ -67,9 +70,11 @@ /// DoPromotion - This method actually performs the promotion of the specified /// arguments, and returns the new function. At this point, we know that it's /// safe to do so. -static CallGraphNode * +static Function * doPromotion(Function *F, SmallPtrSetImpl &ArgsToPromote, - SmallPtrSetImpl &ByValArgsToTransform, CallGraph &CG) { + SmallPtrSetImpl &ByValArgsToTransform, + Optional> + ReplaceCallSite) { // Start by computing a new prototype for the function, which is the same as // the old function, but has modified arguments. @@ -207,9 +212,6 @@ F->getParent()->getFunctionList().insert(F->getIterator(), NF); NF->takeName(F); - // Get a new callgraph node for NF. - CallGraphNode *NF_CGN = CG.getOrInsertFunction(NF); - // Loop over all of the callers of the function, transforming the call sites // to pass in the loaded pointers. // @@ -334,8 +336,8 @@ AttributesVec.clear(); // Update the callgraph to know that the callsite has been transformed. - CallGraphNode *CalleeNode = CG[Call->getParent()->getParent()]; - CalleeNode->replaceCallEdge(CS, CallSite(New), NF_CGN); + if (ReplaceCallSite) + (*ReplaceCallSite)(CS, CallSite(New)); if (!Call->use_empty()) { Call->replaceAllUsesWith(New); @@ -463,18 +465,7 @@ std::advance(I2, ArgIndices.size()); } - NF_CGN->stealCalledFunctionsFrom(CG[F]); - - // Now that the old function is dead, delete it. If there is a dangling - // reference to the CallgraphNode, just leave the dead function around for - // someone else to nuke. - CallGraphNode *CGN = CG[F]; - if (CGN->getNumReferences() == 0) - delete CG.removeFunctionFromModule(CGN); - else - F->setLinkage(Function::ExternalLinkage); - - return NF_CGN; + return NF; } /// AllCallersPassInValidPointerForArgument - Return true if we can prove that @@ -818,14 +809,13 @@ /// example, all callers are direct). If safe to promote some arguments, it /// calls the DoPromotion method. /// -static CallGraphNode * -promoteArguments(CallGraphNode *CGN, CallGraph &CG, - function_ref AARGetter, - unsigned MaxElements) { - Function *F = CGN->getFunction(); - +static Function * +promoteArguments(Function *F, function_ref AARGetter, + unsigned MaxElements, + Optional> + ReplaceCallSite) { // Make sure that it is local to this module. - if (!F || !F->hasLocalLinkage()) + if (!F->hasLocalLinkage()) return nullptr; // Don't promote arguments for variadic functions. Adding, removing, or @@ -950,7 +940,52 @@ if (ArgsToPromote.empty() && ByValArgsToTransform.empty()) return nullptr; - return doPromotion(F, ArgsToPromote, ByValArgsToTransform, CG); + return doPromotion(F, ArgsToPromote, ByValArgsToTransform, ReplaceCallSite); +} + +PreservedAnalyses ArgumentPromotionPass::run(LazyCallGraph::SCC &C, + CGSCCAnalysisManager &AM, + LazyCallGraph &CG, + CGSCCUpdateResult &UR) { + bool Changed = false, LocalChange; + + // Iterate until we stop promoting from this SCC. + do { + LocalChange = false; + + for (LazyCallGraph::Node &N : C) { + Function &OldF = N.getFunction(); + + FunctionAnalysisManager &FAM = + AM.getResult(C, CG).getManager(); + // FIXME: This lambda must only be used with this function. We should + // skip the lambda and just get the AA results directly. + auto AARGetter = [&](Function &F) -> AAResults & { + assert(&F == &OldF && "Called with an unexpected function!"); + return FAM.getResult(F); + }; + + Function *NewF = promoteArguments(&OldF, AARGetter, 3u, None); + if (!NewF) + continue; + LocalChange = true; + + // Directly substitute the functions in the call graph. Note that this + // requires the old function to be completely dead and completely + // replaced by the new function. It does no call graph updates, it merely + // swaps out the particular function mapped to a particular node in the + // graph. + C.getOuterRefSCC().replaceNodeFunction(N, *NewF); + OldF.eraseFromParent(); + } + + Changed |= LocalChange; + } while (LocalChange); + + if (!Changed) + return PreservedAnalyses::all(); + + return PreservedAnalyses::none(); } namespace { @@ -1010,9 +1045,31 @@ LocalChange = false; // Attempt to promote arguments from all functions in this SCC. for (CallGraphNode *OldNode : SCC) { - if (CallGraphNode *NewNode = - promoteArguments(OldNode, CG, AARGetter, MaxElements)) { + Function *OldF = OldNode->getFunction(); + if (!OldF) + continue; + + auto ReplaceCallSite = [&](CallSite OldCS, CallSite NewCS) { + Function *Caller = OldCS.getInstruction()->getParent()->getParent(); + CallGraphNode *NewCalleeNode = + CG.getOrInsertFunction(NewCS.getCalledFunction()); + CallGraphNode *CallerNode = CG[Caller]; + CallerNode->replaceCallEdge(OldCS, NewCS, NewCalleeNode); + }; + + if (Function *NewF = promoteArguments(OldF, AARGetter, MaxElements, + {ReplaceCallSite})) { LocalChange = true; + + // Update the call graph for the newly promoted function. + CallGraphNode *NewNode = CG.getOrInsertFunction(NewF); + NewNode->stealCalledFunctionsFrom(OldNode); + if (OldNode->getNumReferences() == 0) + delete CG.removeFunctionFromModule(OldNode); + else + OldF->setLinkage(Function::ExternalLinkage); + + // And updat ethe SCC we're iterating as well. SCC.ReplaceNode(OldNode, NewNode); } } Index: llvm/trunk/test/Transforms/ArgumentPromotion/aggregate-promote.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/aggregate-promote.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/aggregate-promote.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s %T = type { i32, i32, i32, i32 } @G = constant %T { i32 0, i32 0, i32 17, i32 25 } Index: llvm/trunk/test/Transforms/ArgumentPromotion/attrs.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/attrs.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/attrs.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s %struct.ss = type { i32, i64 } Index: llvm/trunk/test/Transforms/ArgumentPromotion/byval-2.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/byval-2.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/byval-2.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; Arg promotion eliminates the struct argument. ; FIXME: Should it eliminate the i32* argument? Index: llvm/trunk/test/Transforms/ArgumentPromotion/byval.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/byval.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/byval.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" Index: llvm/trunk/test/Transforms/ArgumentPromotion/chained.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/chained.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/chained.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s @G1 = constant i32 0 @G2 = constant i32* @G1 Index: llvm/trunk/test/Transforms/ArgumentPromotion/control-flow.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/control-flow.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/control-flow.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; Don't promote around control flow. define internal i32 @callee(i1 %C, i32* %P) { Index: llvm/trunk/test/Transforms/ArgumentPromotion/control-flow2.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/control-flow2.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/control-flow2.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; CHECK: load i32, i32* %A target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" Index: llvm/trunk/test/Transforms/ArgumentPromotion/crash.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/crash.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/crash.ll @@ -1,4 +1,5 @@ ; RUN: opt -S < %s -inline -argpromotion | FileCheck %s +; RUN: opt -S < %s -passes=inline,argpromotion | FileCheck %s %S = type { %S* } Index: llvm/trunk/test/Transforms/ArgumentPromotion/dbg.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/dbg.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/dbg.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s declare void @sink(i32) Index: llvm/trunk/test/Transforms/ArgumentPromotion/fp80.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/fp80.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/fp80.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/trunk/test/Transforms/ArgumentPromotion/inalloca.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/inalloca.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/inalloca.ll @@ -1,4 +1,5 @@ ; RUN: opt %s -argpromotion -sroa -S | FileCheck %s +; RUN: opt %s -passes='argpromotion,function(sroa)' -S | FileCheck %s target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" Index: llvm/trunk/test/Transforms/ArgumentPromotion/pr27568.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/pr27568.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/pr27568.ll @@ -1,4 +1,5 @@ ; RUN: opt -S -argpromotion < %s | FileCheck %s +; RUN: opt -S -passes=argpromotion < %s | FileCheck %s target triple = "x86_64-pc-windows-msvc" define internal void @callee(i8*) { Index: llvm/trunk/test/Transforms/ArgumentPromotion/reserve-tbaa.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/reserve-tbaa.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/reserve-tbaa.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; PR17906 ; When we promote two arguments in a single function with different types, Index: llvm/trunk/test/Transforms/ArgumentPromotion/sret.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/sret.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/sret.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-windows-msvc" Index: llvm/trunk/test/Transforms/ArgumentPromotion/tail.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/tail.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/tail.ll @@ -1,4 +1,5 @@ ; RUN: opt %s -argpromotion -S -o - | FileCheck %s +; RUN: opt %s -passes=argpromotion -S -o - | FileCheck %s ; PR14710 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" Index: llvm/trunk/test/Transforms/ArgumentPromotion/variadic.ll =================================================================== --- llvm/trunk/test/Transforms/ArgumentPromotion/variadic.ll +++ llvm/trunk/test/Transforms/ArgumentPromotion/variadic.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; Unused arguments from variadic functions cannot be eliminated as that changes ; their classiciation according to the SysV amd64 ABI. Clang and other frontends