Index: include/clang/Driver/Options.td =================================================================== --- include/clang/Driver/Options.td +++ include/clang/Driver/Options.td @@ -752,6 +752,12 @@ def fprofile_generate_EQ : Joined<["-"], "fprofile-generate=">, Group, Flags<[DriverOption]>, MetaVarName<"">, HelpText<"Generate instrumented code to collect execution counts into /default.profraw (overridden by LLVM_PROFILE_FILE env var)">; +def fcs_profile_generate : Flag<["-"], "fcs-profile-generate">, + Group, Flags<[DriverOption]>, + HelpText<"Generate instrumented code to collect context sensitive execution counts into default.profraw (overridden by LLVM_PROFILE_FILE env var)">; +def fcs_profile_generate_EQ : Joined<["-"], "fcs-profile-generate=">, + Group, Flags<[DriverOption]>, MetaVarName<"">, + HelpText<"Generate instrumented code to collect context sensitive execution counts into /default.profraw (overridden by LLVM_PROFILE_FILE env var)">; def fprofile_use : Flag<["-"], "fprofile-use">, Group, Alias; def fprofile_use_EQ : Joined<["-"], "fprofile-use=">, Index: include/clang/Frontend/CodeGenOptions.h =================================================================== --- include/clang/Frontend/CodeGenOptions.h +++ include/clang/Frontend/CodeGenOptions.h @@ -101,6 +101,7 @@ ProfileClangInstr, // Clang instrumentation to generate execution counts // to use with PGO. ProfileIRInstr, // IR level PGO instrumentation in LLVM. + ProfileCSIRInstr, // IR level PGO context sensitive instrumentation in LLVM. }; enum EmbedBitcodeKind { @@ -200,8 +201,8 @@ /// A list of linker options to embed in the object file. std::vector LinkerOptions; - /// Name of the profile file to use as output for -fprofile-instr-generate - /// and -fprofile-generate. + /// Name of the profile file to use as output for -fprofile-instr-generate, + /// -fprofile-generate, and -fcs-profile-generate. std::string InstrProfileOutput; /// Name of the profile file to use with -fprofile-sample-use. @@ -310,6 +311,11 @@ return getProfileInstr() == ProfileIRInstr; } + /// Check if CS IR level profile instrumentation is on. + bool hasProfileCSIRInstr() const { + return getProfileInstr() == ProfileCSIRInstr; + } + /// Check if Clang profile use is on. bool hasProfileClangUse() const { return getProfileUse() == ProfileClangInstr; @@ -317,9 +323,12 @@ /// Check if IR level profile use is on. bool hasProfileIRUse() const { - return getProfileUse() == ProfileIRInstr; + return getProfileUse() == ProfileIRInstr || + getProfileUse() == ProfileCSIRInstr; } + /// Check if IR level profile use is on. + bool hasProfileCSIRUse() const { return getProfileUse() == ProfileCSIRInstr; } }; } // end namespace clang Index: lib/CodeGen/BackendUtil.cpp =================================================================== --- lib/CodeGen/BackendUtil.cpp +++ lib/CodeGen/BackendUtil.cpp @@ -682,15 +682,26 @@ MPM.add(createInstrProfilingLegacyPass(Options)); } + bool hasIRInstr = false; if (CodeGenOpts.hasProfileIRInstr()) { PMBuilder.EnablePGOInstrGen = true; + hasIRInstr = true; + } + if (CodeGenOpts.hasProfileCSIRInstr()) { + assert(!hasIRInstr && "Cannot have both IR and CSIR instrumentation"); + PMBuilder.EnablePGOCSInstrGen = true; + hasIRInstr = true; + } + if (hasIRInstr) { if (!CodeGenOpts.InstrProfileOutput.empty()) PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput; else PMBuilder.PGOInstrGen = DefaultProfileGenName; } - if (CodeGenOpts.hasProfileIRUse()) + if (CodeGenOpts.hasProfileIRUse()) { PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath; + PMBuilder.EnablePGOCSInstrUse = CodeGenOpts.hasProfileCSIRUse(); + } if (!CodeGenOpts.SampleProfileFile.empty()) PMBuilder.PGOSampleUse = CodeGenOpts.SampleProfileFile; @@ -931,24 +942,42 @@ if (CodeGenOpts.hasProfileIRInstr()) // -fprofile-generate. - PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty() - ? DefaultProfileGenName - : CodeGenOpts.InstrProfileOutput, - "", "", "", true, - CodeGenOpts.DebugInfoForProfiling); - else if (CodeGenOpts.hasProfileIRUse()) + PGOOpt = PGOOptions( + CodeGenOpts.InstrProfileOutput.empty() ? DefaultProfileGenName + : CodeGenOpts.InstrProfileOutput, + "", "", "", PGOOptions::IRInstr, CodeGenOpts.DebugInfoForProfiling); + else if (CodeGenOpts.hasProfileIRUse()) { // -fprofile-use. + auto Action = CodeGenOpts.hasProfileCSIRUse() ? PGOOptions::CSIRUse + : PGOOptions::NoAction; PGOOpt = PGOOptions("", CodeGenOpts.ProfileInstrumentUsePath, "", - CodeGenOpts.ProfileRemappingFile, false, + CodeGenOpts.ProfileRemappingFile, Action, CodeGenOpts.DebugInfoForProfiling); - else if (!CodeGenOpts.SampleProfileFile.empty()) + } else if (!CodeGenOpts.SampleProfileFile.empty()) // -fprofile-sample-use PGOOpt = PGOOptions("", "", CodeGenOpts.SampleProfileFile, - CodeGenOpts.ProfileRemappingFile, false, + CodeGenOpts.ProfileRemappingFile, PGOOptions::NoAction, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.DebugInfoForProfiling) // -fdebug-info-for-profiling - PGOOpt = PGOOptions("", "", "", "", false, true); + PGOOpt = PGOOptions("", "", "", "", PGOOptions::NoAction, true); + + // Check to see if we want to generate a CS profile. + if (CodeGenOpts.hasProfileCSIRInstr()) { + assert(!CodeGenOpts.hasProfileCSIRUse() && + "Cannot have both CSProfileUse and CSProfileGen at the same time"); + if (PGOOpt.hasValue()) { + assert((PGOOpt->ProfileGenFile == "") && + "Cannot have both ProfileGen and CSProfileGen at the same time"); + PGOOpt->ProfileGenFile = CodeGenOpts.InstrProfileOutput; + PGOOpt->Action = PGOOptions::CSIRInstr; + } else + PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty() + ? DefaultProfileGenName + : CodeGenOpts.InstrProfileOutput, + "", "", "", PGOOptions::CSIRInstr, + CodeGenOpts.DebugInfoForProfiling); + } PassBuilder PB(TM.get(), PGOOpt); @@ -1210,6 +1239,16 @@ Conf.CGOptLevel = getCGOptLevel(CGOpts); initTargetOptions(Conf.Options, CGOpts, TOpts, LOpts, HeaderOpts); Conf.SampleProfile = std::move(SampleProfile); + + // Context sensitive profile. + if (CGOpts.hasProfileCSIRInstr()) { + Conf.RunCSIRInstr = true; + Conf.CSIRProfile = std::move(CGOpts.InstrProfileOutput); + } else if (CGOpts.hasProfileCSIRUse()) { + Conf.RunCSIRInstr = false; + Conf.CSIRProfile = std::move(CGOpts.ProfileInstrumentUsePath); + } + Conf.ProfileRemapping = std::move(ProfileRemapping); Conf.UseNewPM = CGOpts.ExperimentalNewPassManager; Conf.DebugPassManager = CGOpts.DebugPassManager; Index: lib/CodeGen/CodeGenModule.cpp =================================================================== --- lib/CodeGen/CodeGenModule.cpp +++ lib/CodeGen/CodeGenModule.cpp @@ -53,6 +53,7 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/ConvertUTF.h" @@ -418,7 +419,8 @@ OpenMPRuntime->clear(); } if (PGOReader) { - getModule().setProfileSummary(PGOReader->getSummary().getMD(VMContext)); + getModule().setProfileSummary(PGOReader->getSummary(false).getMD(VMContext), + llvm::ProfileSummary::PSK_Instr); if (PGOStats.hasDiagnostics()) PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName); } Index: lib/Driver/ToolChain.cpp =================================================================== --- lib/Driver/ToolChain.cpp +++ lib/Driver/ToolChain.cpp @@ -407,6 +407,8 @@ false) || Args.hasArg(options::OPT_fprofile_generate) || Args.hasArg(options::OPT_fprofile_generate_EQ) || + Args.hasArg(options::OPT_fcs_profile_generate) || + Args.hasArg(options::OPT_fcs_profile_generate_EQ) || Args.hasArg(options::OPT_fprofile_instr_generate) || Args.hasArg(options::OPT_fprofile_instr_generate_EQ) || Args.hasArg(options::OPT_fcreate_profile) || Index: lib/Driver/ToolChains/Clang.cpp =================================================================== --- lib/Driver/ToolChains/Clang.cpp +++ lib/Driver/ToolChains/Clang.cpp @@ -725,6 +725,13 @@ PGOGenerateArg->getOption().matches(options::OPT_fno_profile_generate)) PGOGenerateArg = nullptr; + auto *CSPGOGenerateArg = Args.getLastArg(options::OPT_fcs_profile_generate, + options::OPT_fcs_profile_generate_EQ, + options::OPT_fno_profile_generate); + if (CSPGOGenerateArg && + CSPGOGenerateArg->getOption().matches(options::OPT_fno_profile_generate)) + CSPGOGenerateArg = nullptr; + auto *ProfileGenerateArg = Args.getLastArg( options::OPT_fprofile_instr_generate, options::OPT_fprofile_instr_generate_EQ, @@ -757,11 +764,22 @@ CmdArgs.push_back("-fprofile-instrument=clang"); } + Arg *PGOGenArg = nullptr; if (PGOGenerateArg) { + assert(!CSPGOGenerateArg); + PGOGenArg = PGOGenerateArg; CmdArgs.push_back("-fprofile-instrument=llvm"); - if (PGOGenerateArg->getOption().matches( - options::OPT_fprofile_generate_EQ)) { - SmallString<128> Path(PGOGenerateArg->getValue()); + } + if (CSPGOGenerateArg) { + assert(!PGOGenerateArg); + PGOGenArg = CSPGOGenerateArg; + CmdArgs.push_back("-fprofile-instrument=csllvm"); + } + if (PGOGenArg) { + if (PGOGenArg->getOption().matches( + PGOGenerateArg ? options::OPT_fprofile_generate_EQ + : options::OPT_fcs_profile_generate_EQ)) { + SmallString<128> Path(PGOGenArg->getValue()); llvm::sys::path::append(Path, "default_%m.profraw"); CmdArgs.push_back( Args.MakeArgString(Twine("-fprofile-instrument-path=") + Path)); Index: lib/Driver/ToolChains/CommonArgs.cpp =================================================================== --- lib/Driver/ToolChains/CommonArgs.cpp +++ lib/Driver/ToolChains/CommonArgs.cpp @@ -463,6 +463,31 @@ Args.MakeArgString(Twine("-plugin-opt=sample-profile=") + FName)); } + auto *CSPGOGenerateArg = Args.getLastArg(options::OPT_fcs_profile_generate, + options::OPT_fcs_profile_generate_EQ, + options::OPT_fno_profile_generate); + if (CSPGOGenerateArg && + CSPGOGenerateArg->getOption().matches(options::OPT_fno_profile_generate)) + CSPGOGenerateArg = nullptr; + + auto *ProfileUseArg = getLastProfileUseArg(Args); + + if (CSPGOGenerateArg) { + CmdArgs.push_back(Args.MakeArgString("-plugin-opt=cs-profile-generate")); + if (CSPGOGenerateArg->getOption().matches( + options::OPT_fcs_profile_generate_EQ)) { + SmallString<128> Path(CSPGOGenerateArg->getValue()); + llvm::sys::path::append(Path, "default_%m.profraw"); + CmdArgs.push_back( + Args.MakeArgString(Twine("-plugin-opt=cs-profile-path=") + Path)); + } else + CmdArgs.push_back( + Args.MakeArgString("-plugin-opt=cs-profile-path=default_%m.profraw")); + } else if (ProfileUseArg) { + CmdArgs.push_back(Args.MakeArgString(Twine("-plugin-opt=cs-profile-path=") + + ProfileUseArg->getValue())); + } + // Need this flag to turn on new pass manager via Gold plugin. if (Args.hasFlag(options::OPT_fexperimental_new_pass_manager, options::OPT_fno_experimental_new_pass_manager, Index: lib/Frontend/CompilerInvocation.cpp =================================================================== --- lib/Frontend/CompilerInvocation.cpp +++ lib/Frontend/CompilerInvocation.cpp @@ -476,6 +476,7 @@ .Case("none", CodeGenOptions::ProfileNone) .Case("clang", CodeGenOptions::ProfileClangInstr) .Case("llvm", CodeGenOptions::ProfileIRInstr) + .Case("csllvm", CodeGenOptions::ProfileCSIRInstr) .Default(~0U); if (I == ~0U) { Diags.Report(diag::err_drv_invalid_pgo_instrumentor) << A->getAsString(Args) @@ -498,9 +499,12 @@ } std::unique_ptr PGOReader = std::move(ReaderOrErr.get()); - if (PGOReader->isIRLevelProfile()) - Opts.setProfileUse(CodeGenOptions::ProfileIRInstr); - else + if (PGOReader->isIRLevelProfile()) { + if (PGOReader->hasCSIRLevelProfile()) + Opts.setProfileUse(CodeGenOptions::ProfileCSIRInstr); + else + Opts.setProfileUse(CodeGenOptions::ProfileIRInstr); + } else Opts.setProfileUse(CodeGenOptions::ProfileClangInstr); } Index: test/CodeGen/Inputs/pgotestir.proftext =================================================================== --- test/CodeGen/Inputs/pgotestir.proftext +++ test/CodeGen/Inputs/pgotestir.proftext @@ -0,0 +1,2 @@ +# IR level Instrumentation Flag +:ir Index: test/CodeGen/Inputs/pgotestir_cs.proftext =================================================================== --- test/CodeGen/Inputs/pgotestir_cs.proftext +++ test/CodeGen/Inputs/pgotestir_cs.proftext @@ -0,0 +1,2 @@ +# IR level Instrumentation Flag with CS +:csir Index: test/CodeGen/cspgo-instrumentation.c =================================================================== --- test/CodeGen/cspgo-instrumentation.c +++ test/CodeGen/cspgo-instrumentation.c @@ -0,0 +1,35 @@ +// Test if CSPGO instrumentation and use pass are invoked. +// +// Ensure Pass PGOInstrumentationGenPass is invoked. +// RUN: %clang_cc1 -O2 -fprofile-instrument=csllvm %s -mllvm -debug-pass=Structure -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN +// RUN: %clang_cc1 -O2 -fprofile-instrument=csllvm %s -fexperimental-new-pass-manager -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN-NEWPM +// CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN: PGOInstrumentationGenPass +// CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN-NEWPM: Running pass: PGOInstrumentationGen +// +// RUN: llvm-profdata merge -o %t.profdata %S/Inputs/pgotestir.proftext +// +// Ensure Pass PGOInstrumentationUsePass and PGOInstrumentationGenPass are invoked. +// RUN: %clang_cc1 -O2 -fprofile-instrument-use-path=%t.profdata -fprofile-instrument=csllvm %s -mllvm -debug-pass=Structure -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN2 +// RUN: %clang_cc1 -O2 -fprofile-instrument-use-path=%t.profdata -fprofile-instrument=csllvm %s -fexperimental-new-pass-manager -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN2-NEWPM +// CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN2: PGOInstrumentationUsePass +// CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN2: PGOInstrumentationGenPass +// CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN2-NEWPM: Running pass: PGOInstrumentationUse +// CHECK-CSPGOGENPASS-INVOKED-INSTR-GEN2-NEWPM: Running pass: PGOInstrumentationGen + +// Ensure Pass PGOInstrumentationUsePass is invoked only once. +// RUN: %clang_cc1 -O2 -fprofile-instrument-use-path=%t.profdata %s -mllvm -debug-pass=Structure -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOUSEPASS-INVOKED-USE +// RUN: %clang_cc1 -O2 -fprofile-instrument-use-path=%t.profdata %s -fexperimental-new-pass-manager -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOUSEPASS-INVOKED-USE-NEWPM +// CHECK-PGOUSEPASS-INVOKED-USE: PGOInstrumentationUsePass +// CHECK-PGOUSEPASS-INVOKED-USE-NOT: PGOInstrumentationUsePass +// CHECK-PGOUSEPASS-INVOKED-USE-NEWPM: Running pass: PGOInstrumentationUse +// CHECK-PGOUSEPASS-INVOKED-USE-NEWPM-NOT: Running pass: PGOInstrumentationUse +// +// Ensure Pass PGOInstrumentationUsePass is invoked twice. +// RUN: llvm-profdata merge -o %t_cs.profdata %S/Inputs/pgotestir_cs.proftext +// RUN: %clang_cc1 -O2 -fprofile-instrument-use-path=%t_cs.profdata %s -mllvm -debug-pass=Structure -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOUSEPASS-INVOKED-USE2 +// RUN: %clang_cc1 -O2 -fprofile-instrument-use-path=%t_cs.profdata %s -fexperimental-new-pass-manager -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOUSEPASS-INVOKED-USE2-NEWPM +// CHECK-PGOUSEPASS-INVOKED-USE2: PGOInstrumentationUsePass +// CHECK-PGOUSEPASS-INVOKED-USE2: PGOInstrumentationUsePass +// CHECK-PGOUSEPASS-INVOKED-USE2-NEWPM: Running pass: PGOInstrumentationUse +// CHECK-PGOUSEPASS-INVOKED-USE2-NEWPM: Running pass: PGOInstrumentationUse +