diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -0,0 +1,65 @@ +//===- Transforms/IPO/SampleProfileProbe.h ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This file provides the interface for the pseudo probe implementation for +/// AutoFDO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H +#define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Target/TargetMachine.h" +#include + +namespace llvm { + +class Module; + +using BlockIdMap = std::unordered_map; + +enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid }; + +enum class PseudoProbeType { Block = 0 }; + +/// Sample profile pseudo prober. +/// +/// Insert pseudo probes for block sampling and value sampling. +class SampleProfileProber { +public: + // Give an empty module id when the prober is not used for instrumentation. + SampleProfileProber(Function &F); + void instrumentOneFunc(Function &F, TargetMachine *TM); + +private: + Function *getFunction() const { return F; } + uint32_t getBlockId(const BasicBlock *BB) const; + void computeProbeIdForBlocks(); + + Function *F; + + /// Map basic blocks to the their pseudo probe ids. + BlockIdMap BlockProbeIds; + + /// The ID of the last probe, Can be used to number a new probe. + uint32_t LastProbeId; +}; + +class SampleProfileProbePass : public PassInfoMixin { + TargetMachine *TM; + +public: + SampleProfileProbePass(TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // end namespace llvm +#endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -97,6 +97,7 @@ #include "llvm/Transforms/IPO/PartialInlining.h" #include "llvm/Transforms/IPO/SCCP.h" #include "llvm/Transforms/IPO/SampleProfile.h" +#include "llvm/Transforms/IPO/SampleProfileProbe.h" #include "llvm/Transforms/IPO/StripDeadPrototypes.h" #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h" #include "llvm/Transforms/IPO/WholeProgramDevirt.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -85,6 +85,7 @@ MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("scc-oz-module-inliner", buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, DebugLogging)) +MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM)) MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass()) MODULE_PASS("synthetic-counts-propagation", SyntheticCountsPropagation()) MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass(nullptr, nullptr)) diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt --- a/llvm/lib/Transforms/IPO/CMakeLists.txt +++ b/llvm/lib/Transforms/IPO/CMakeLists.txt @@ -31,6 +31,7 @@ PassManagerBuilder.cpp PruneEH.cpp SampleProfile.cpp + SampleProfileProbe.cpp SCCP.cpp StripDeadPrototypes.cpp StripSymbols.cpp diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp @@ -0,0 +1,98 @@ +//===- SampleProfileProbe.cpp - Pseudo probe Instrumentation -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the SampleProfileProber transformation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/IPO/SampleProfileProbe.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/CRC.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include + +using namespace llvm; +#define DEBUG_TYPE "sample-profile-probe" + +SampleProfileProber::SampleProfileProber(Function &Func) : F(&Func) { + BlockProbeIds.clear(); + LastProbeId = (uint32_t)PseudoProbeReservedId::Last; + computeProbeIdForBlocks(); +} + +void SampleProfileProber::computeProbeIdForBlocks() { + for (auto &BB : *F) { + BlockProbeIds[&BB] = ++LastProbeId; + } +} + +uint32_t SampleProfileProber::getBlockId(const BasicBlock *BB) const { + auto I = BlockProbeIds.find(const_cast(BB)); + return I == BlockProbeIds.end() ? 0 : I->second; +} + +void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) { + Module *M = F.getParent(); + MDBuilder MDB(F.getContext()); + // Compute a GUID without considering the function's linkage type. This is + // fine since function name is the only key in the profile database. + uint64_t Guid = Function::getGUID(F.getName()); + + // Probe basic blocks. + for (auto &I : BlockProbeIds) { + BasicBlock *BB = I.first; + uint32_t Index = I.second; + // Insert a probe before an instruction with a valid debug line number which + // will be assigned to the probe. The line number will be used later to + // model the inline context when the probe is inlined into other functions. + // Debug instructions, phi nodes and lifetime markers do not have an valid + // line number. Real instructions generated by optimizations may not come + // with a line number either. + auto HasValidDbgLine = [](Instruction *J) { + return !isa(J) && !isa(J) && + !J->isLifetimeStartOrEnd() && J->getDebugLoc(); + }; + + Instruction *J = &*BB->getFirstInsertionPt(); + while (J != BB->getTerminator() && !HasValidDbgLine(J)) { + J = J->getNextNode(); + } + + IRBuilder<> Builder(J); + assert(Builder.GetInsertPoint() != BB->end() && + "Cannot get the probing point"); + Function *ProbeFn = + llvm::Intrinsic::getDeclaration(M, Intrinsic::pseudoprobe); + Value *Args[] = {Builder.getInt64(Guid), Builder.getInt64(Index)}; + Builder.CreateCall(ProbeFn, Args); + } +} + +PreservedAnalyses SampleProfileProbePass::run(Module &M, + ModuleAnalysisManager &AM) { + for (auto &F : M) { + if (F.isDeclaration()) + continue; + SampleProfileProber ProbeManager(F); + ProbeManager.instrumentOneFunc(F, TM); + } + + return PreservedAnalyses::none(); +} diff --git a/llvm/test/Transforms/SampleProfile/emit-pseudo-probe.ll b/llvm/test/Transforms/SampleProfile/emit-pseudo-probe.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/emit-pseudo-probe.ll @@ -0,0 +1,29 @@ +; RUN: opt < %s -passes=pseudo-probe -function-sections -S -o %t +; RUN: FileCheck %s < %t --check-prefix=CHECK-IL +; RUN: llc %t -stop-after=instruction-select -o - | FileCheck %s --check-prefix=CHECK-MIR +; +;; Check the generation of pseudoprobe intrinsic call. + +define void @foo(i32 %x) { +bb0: + %cmp = icmp eq i32 %x, 0 +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 1) +; CHECK-MIR: PSEUDO_PROBE [[#GUID:]], 1, 0 + br i1 %cmp, label %bb1, label %bb2 + +bb1: +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 2) +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 3, 0 +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0 + br label %bb3 + +bb2: +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 3) +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 2, 0 +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0 + br label %bb3 + +bb3: +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 4) + ret void +}