diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -19,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/AsmPrinterHandler.h" #include "llvm/CodeGen/DwarfStringPoolEntry.h" +#include "llvm/CodeGen/MIPSectionEmitter.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/LLVMContext.h" @@ -96,6 +97,9 @@ /// generating (such as the current section etc). std::unique_ptr OutStreamer; + /// Used to emit special sections for machine instrumentation. + MIPSectionEmitter MIPEmitter; + /// The current machine function. MachineFunction *MF = nullptr; diff --git a/llvm/include/llvm/CodeGen/MIPSectionEmitter.h b/llvm/include/llvm/CodeGen/MIPSectionEmitter.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MIPSectionEmitter.h @@ -0,0 +1,70 @@ +//===- MIPSectionEmitter.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MIPSECTIONEMITTER_H +#define LLVM_CODEGEN_MIPSECTIONEMITTER_H + +#include "MachineFunction.h" +#include "MachineOperand.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/Module.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/MIP/MIP.h" +#include "llvm/Support/Endian.h" +#include + +namespace llvm { + +class AsmPrinter; + +class MIPSectionEmitter { +public: + explicit MIPSectionEmitter(AsmPrinter &AP); + + void runOnMachineFunctionStart(MachineFunction &MF); + void runOnMachineFunctionEnd(MachineFunction &MF); + void runOnFunctionInstrumentationMarker(const MachineInstr &MI); + void runOnBasicBlockInstrumentationMarker(const MachineInstr &MI); + void serializeToMIPRawSection(); + void serializeToMIPMapSection(); + + MCSymbol *getRawProfileSymbol(); + uint64_t getOffsetToRawBlockProfileSymbol(uint32_t BlockID); + +private: + struct MBBInfo { + const MCSymbol *StartSymbol; + }; + + struct MFInfo { + const Function *Func; + const MCSymbol *StartSymbol; + const MCSymbol *EndSymbol; + MCSymbol *RawProfileSymbol; + uint32_t ControlFlowGraphSignature; + uint32_t NonEntryBasicBlockCount; + + // A map from Machine Basic Block IDs to MBBInfo. + DenseMap BasicBlockInfos; + }; + + void emitMIPHeader(MachineProfile::MIPFileType FileType); + void emitMIPFunctionData(const MFInfo &Info); + void emitMIPFunctionInfo(MFInfo &Info); + MCSymbol *getMIPSectionBeginSymbol(Twine MIPSectionName); + + AsmPrinter &AP; + MCSymbol *CurrentFunctionEndSymbol; + + // A map from a function symbol to its function info. + std::map FunctionInfos; +}; + +} // end namespace llvm + +#endif // LLVM_CODEGEN_MIPSECTIONEMITTER_H diff --git a/llvm/include/llvm/CodeGen/MIRInstrumentationPass.h b/llvm/include/llvm/CodeGen/MIRInstrumentationPass.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MIRInstrumentationPass.h @@ -0,0 +1,54 @@ +//===-------------- MIRInstrumentationPass.h ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MIRINSTRUMENTATIONPASS_H +#define LLVM_CODEGEN_MIRINSTRUMENTATIONPASS_H + +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Support/CommandLine.h" + +namespace llvm { + +class MIRInstrumentation : public MachineFunctionPass { +public: + static char ID; + MIRInstrumentation() : MachineFunctionPass(ID) {} + + static cl::opt EnableMachineInstrumentation; + static cl::opt EnableMachineFunctionCoverage; + static cl::opt EnableMachineBasicBlockCoverage; + static cl::opt EnableMachineCallGraph; + static cl::opt MachineProfileRuntimeBufferSize; + static cl::opt MachineProfileFunctionGroupCount; + static cl::opt MachineProfileSelectedFunctionGroup; + static cl::opt MachineProfileMinInstructionSize; + static std::string LinkUnitName; + static cl::opt LinkUnitNameOption; + +private: + StringRef getPassName() const override { + return "Add instrumentation code to machine functions."; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + bool doInitialization(Module &M) override; + bool shouldInstrumentMachineFunction(const MachineFunction &MF) const; + bool runOnMachineFunction(MachineFunction &MF) override; + uint32_t getControlFlowGraphSignature( + SmallVectorImpl &MBBs) const; + void getMachineBasicBlocks(MachineFunction &MF, + SmallVectorImpl &MBBs) const; + void runOnMachineBasicBlock(MachineBasicBlock &MBB, uint32_t BlockID); +}; +} // namespace llvm + +#endif // LLVM_CODEGEN_MIRINSTRUMENTATIONPASS_H diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -1320,6 +1320,7 @@ case TargetOpcode::LIFETIME_START: case TargetOpcode::LIFETIME_END: case TargetOpcode::PSEUDO_PROBE: + case TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER: return true; } } diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -329,6 +329,9 @@ /// the target platform. extern char &XRayInstrumentationID; + /// This pass injects instrumentation code into machine functions. + extern char &MIRInstrumentationID; + /// This pass inserts FEntry calls extern char &FEntryInserterID; diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1987,6 +1987,14 @@ return MI.getOperand(0); } + /// Returns a target-specific temporary register that is dead at the + /// beginning of the given machine basic block for machine profile + /// instrumentation. + virtual Register + getTemporaryMachineProfileRegister(const MachineBasicBlock &MBB) const { + return MCRegister::NoRegister; + } + private: mutable std::unique_ptr Formatter; unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode; diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -277,6 +277,7 @@ void initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(PassRegistry &); void initializeMIRAddFSDiscriminatorsPass(PassRegistry &); void initializeMIRCanonicalizerPass(PassRegistry &); +void initializeMIRInstrumentationPass(PassRegistry &); void initializeMIRNamerPass(PassRegistry &); void initializeMIRPrintingPassPass(PassRegistry&); void initializeMachineBlockFrequencyInfoPass(PassRegistry&); diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -163,6 +163,15 @@ /// FaultMap section. MCSection *FaultMapSection = nullptr; + /// Specials sections for machine instrumentation. + MCSection *MIPRawSection = nullptr; + MCSection *MIPMapSection = nullptr; + + /// Special COMDAT sections for machine instrumentation. Only used on ELF + /// targets. + MCSection *MIPRawHeaderComdatSection = nullptr; + MCSection *MIPMapHeaderComdatSection = nullptr; + /// Remarks section. MCSection *RemarksSection = nullptr; @@ -340,6 +349,16 @@ MCSection *getStackMapSection() const { return StackMapSection; } MCSection *getFaultMapSection() const { return FaultMapSection; } + + MCSection *getMIPRawSection() const { return MIPRawSection; } + MCSection *getMIPMapSection() const { return MIPMapSection; } + MCSection *getMIPRawHeaderComdatSection() const { + return MIPRawHeaderComdatSection; + } + MCSection *getMIPMapHeaderComdatSection() const { + return MIPMapHeaderComdatSection; + } + MCSection *getRemarksSection() const { return RemarksSection; } MCSection *getStackSizesSection(const MCSection &TextSec) const; diff --git a/llvm/include/llvm/MIP/MIP.h b/llvm/include/llvm/MIP/MIP.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/MIP/MIP.h @@ -0,0 +1,20 @@ +//===- MIP.h - Machine IR Profile -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MIP_MIP_H +#define LLVM_MIP_MIP_H + +namespace llvm { +namespace MachineProfile { + +#include "llvm/MIP/MIPData.inc" + +} // namespace MachineProfile +} // namespace llvm + +#endif // LLVM_MIP_MIP_H diff --git a/llvm/include/llvm/MIP/MIPData.inc b/llvm/include/llvm/MIP/MIPData.inc new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/MIP/MIPData.inc @@ -0,0 +1,77 @@ +//===-- MIPData.inc - machine ir profile runtime structures -------*- C -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains definitions and data structures that are shared between +// the runtime and the compiler. It lives in two locations that need to stay in +// sync. +// + llvm/include/llvm/MIP/MIPData.inc +// + compiler-rt/include/mip/MIPData.inc +// +//===----------------------------------------------------------------------===// + +#ifndef MIP_DATA_DEFINED + +#include + +#ifndef MIP_DATA_INC +#define MIP_DATA_INC + +#define MIP_SIMPLE_QUOTE(x) #x +#define MIP_QUOTE(x) MIP_SIMPLE_QUOTE(x) +#define MIP_SIMPLE_CONCAT(x,y) x ## y +#define MIP_CONCAT(x,y) MIP_SIMPLE_CONCAT(x,y) + +#define MIP_RAW_SECTION __llvm_mipraw +#define MIP_MAP_SECTION __llvm_mipmap +#define MIP_RUNTIME_SYMBOL __llvm_mip_runtime + +#define MIP_RAW_SECTION_NAME MIP_QUOTE(MIP_RAW_SECTION) +#define MIP_MAP_SECTION_NAME MIP_QUOTE(MIP_MAP_SECTION) +#define MIP_RUNTIME_SYMBOL_NAME MIP_QUOTE(MIP_RUNTIME_SYMBOL) + +// MIP magic value in little endian format. +// \251 M I P +// 0xFB 0x4D 0x49 0x50 +#define MIP_MAGIC_VALUE (0x50494DFB) +#define MIP_VERSION (8) + +typedef enum { + MIP_FILE_TYPE_RAW = 0x0001, // .mipraw + MIP_FILE_TYPE_MAP = 0x0002, // .mipmap + MIP_FILE_TYPE_PROFILE = 0x0003, // .mip + MIP_FILE_TYPE_CALL_EDGE_SAMPLES = 0x0004, // .mipret +} MIPFileType; + +typedef enum { + MIP_PROFILE_TYPE_FUNCTION_COVERAGE = 1 << 0, + MIP_PROFILE_TYPE_BLOCK_COVERAGE = 1 << 1, + MIP_PROFILE_TYPE_FUNCTION_TIMESTAMP = 1 << 2, + MIP_PROFILE_TYPE_FUNCTION_CALL_COUNT = 1 << 3, + MIP_PROFILE_TYPE_RETURN_ADDRESS = 1 << 4, +} MIPProfileType; + +typedef struct { + uint32_t Magic; + uint16_t Version; + uint16_t FileType; + uint32_t ProfileType; + uint32_t ModuleHash; + uint32_t Reserved; + uint32_t OffsetToData; +} MIPHeader; + +typedef struct { + uint32_t CalleeProfileDataOffset; + uint32_t SectionRelativeReturnAddress; +} CallEdge_t; + +#endif // MIP_DATA_INC + +#else // MIP_DATA_INC +#undef MIP_DATA_DEFINED +#endif // MIP_DATA_DEFINED diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -34,6 +34,21 @@ HANDLE_TARGET_OPCODE(GC_LABEL) HANDLE_TARGET_OPCODE(ANNOTATION_LABEL) +/// MIP_FUNCTION_INSTRUMENTATION_MARKER - This instruction is a noop that marks +/// this function as an instrumented function. It takes as input the control +/// flow graph signature and the number of machine basic blocks. +HANDLE_TARGET_OPCODE(MIP_FUNCTION_INSTRUMENTATION_MARKER) + +/// MIP_FUNCTION_COVERAGE_INSTRUMENTATION - This instruction implements function +/// coverage instrumentation. It takes as input a free temporary register if +/// one exists. +HANDLE_TARGET_OPCODE(MIP_FUNCTION_COVERAGE_INSTRUMENTATION) + +/// MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION - This instruction implements basic +/// block coverage instrumentation. It takes as input a free temporary register +/// if one exists and the id of the instrumented block. +HANDLE_TARGET_OPCODE(MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION) + /// KILL - This instruction is a noop that is used only to adjust the /// liveness of registers. This can be useful when dealing with /// sub-registers. diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -1014,6 +1014,24 @@ let hasNoSchedulingInfo = true; let Namespace = "TargetOpcode"; } +def MIP_FUNCTION_INSTRUMENTATION_MARKER : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i32imm:$cfg, i32imm:$n); + let AsmString = ""; + let hasSideEffects = 0; +} +def MIP_FUNCTION_COVERAGE_INSTRUMENTATION : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins unknown:$reg); + let AsmString = ""; + let hasSideEffects = 1; +} +def MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins unknown:$reg, i32imm:$n); + let AsmString = ""; + let hasSideEffects = 1; +} def PHI : StandardPseudoInstruction { let OutOperandList = (outs unknown:$dst); let InOperandList = (ins variable_ops); diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -195,7 +195,8 @@ AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr Streamer) : MachineFunctionPass(ID), TM(tm), MAI(tm.getMCAsmInfo()), - OutContext(Streamer->getContext()), OutStreamer(std::move(Streamer)) { + OutContext(Streamer->getContext()), OutStreamer(std::move(Streamer)), + MIPEmitter(*this) { VerboseAsm = OutStreamer->isVerboseAsm(); } @@ -1464,6 +1465,8 @@ OutStreamer->emitLabel(CurrentFnEnd); } + MIPEmitter.runOnMachineFunctionEnd(*MF); + // If the target wants a .size directive for the size of the function, emit // it. if (MAI->hasDotTypeDotSizeDirective()) { @@ -1925,6 +1928,9 @@ // after everything else has gone out. emitEndOfAsmFile(M); + MIPEmitter.serializeToMIPRawSection(); + MIPEmitter.serializeToMIPMapSection(); + MMI = nullptr; OutStreamer->Finish(); @@ -1979,6 +1985,7 @@ } ORE = &getAnalysis().getORE(); + MIPEmitter.runOnMachineFunctionStart(MF); } namespace { diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -146,6 +146,8 @@ MIRVRegNamerUtils.cpp MIRNamerPass.cpp MIRCanonicalizerPass.cpp + MIRInstrumentationPass.cpp + MIPSectionEmitter.cpp RegisterUsageInfo.cpp RegUsageInfoCollector.cpp RegUsageInfoPropagate.cpp diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -59,6 +59,7 @@ initializeLocalStackSlotPassPass(Registry); initializeLowerIntrinsicsPass(Registry); initializeMIRCanonicalizerPass(Registry); + initializeMIRInstrumentationPass(Registry); initializeMIRNamerPass(Registry); initializeMachineBlockFrequencyInfoPass(Registry); initializeMachineBlockPlacementPass(Registry); diff --git a/llvm/lib/CodeGen/MIPSectionEmitter.cpp b/llvm/lib/CodeGen/MIPSectionEmitter.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/MIPSectionEmitter.cpp @@ -0,0 +1,349 @@ +//===- MIPSectionEmitter.cpp ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MIPSectionEmitter.h" +#include "llvm/ADT/Twine.h" +#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/MIRInstrumentationPass.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/Mangler.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCValue.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; +using namespace llvm::MachineProfile; + +std::string getMangledName(const Function *F) { + std::string MangledName; + raw_string_ostream MangledNameOS(MangledName); + Mangler().getNameWithPrefix(MangledNameOS, F, + /*CannotUsePrivateLabel=*/true); + return MangledName; +} + +MIPSectionEmitter::MIPSectionEmitter(AsmPrinter &AP) : AP(AP) {} + +MCSymbol *MIPSectionEmitter::getMIPSectionBeginSymbol(Twine MIPSectionName) { + auto &OS = *AP.OutStreamer; + auto &OutContext = OS.getContext(); + const auto &TT = AP.TM.getTargetTriple(); + + if (TT.getObjectFormat() == Triple::ELF) + return OutContext.getOrCreateSymbol("__start_" + MIPSectionName); + if (TT.getObjectFormat() == Triple::MachO) + return OutContext.getOrCreateSymbol("__header$" + MIPSectionName); + llvm_unreachable("Unsupported target triple"); +} + +void MIPSectionEmitter::runOnMachineFunctionStart(MachineFunction &MF) { + if (!MIRInstrumentation::EnableMachineInstrumentation) + return; + + for (auto &MBB : MF) + MBB.setLabelMustBeEmitted(); + CurrentFunctionEndSymbol = AP.createTempSymbol("mip_func_end"); +} + +void MIPSectionEmitter::runOnMachineFunctionEnd(MachineFunction &MF) { + if (!MIRInstrumentation::EnableMachineInstrumentation) + return; + + auto &OS = *AP.OutStreamer; + OS.emitLabel(CurrentFunctionEndSymbol); +} + +void MIPSectionEmitter::runOnFunctionInstrumentationMarker( + const MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER); + auto &OS = *AP.OutStreamer; + auto &OutContext = OS.getContext(); + + MFInfo Info; + Info.Func = &AP.MF->getFunction(); + Info.StartSymbol = AP.getSymbol(Info.Func); + Info.EndSymbol = CurrentFunctionEndSymbol; + Info.RawProfileSymbol = + OutContext.getOrCreateSymbol(getMangledName(Info.Func) + "$RAW"); + Info.ControlFlowGraphSignature = MI.getOperand(0).getImm(); + Info.NonEntryBasicBlockCount = MI.getOperand(1).getImm(); + + FunctionInfos.insert(std::make_pair(Info.StartSymbol, Info)); +} + +void MIPSectionEmitter::runOnBasicBlockInstrumentationMarker( + const MachineInstr &MI) { + assert(MI.getOpcode() == + TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION); + + auto BlockID = MI.getOperand(1).getImm(); + + MBBInfo Info; + Info.StartSymbol = MI.getParent()->getSymbol(); + + // TODO: Properly lookup the correct function info instead of just looking at + // the function we are currently in. + auto *FunctionSymbol = AP.getSymbol(&AP.MF->getFunction()); + auto &FunctionInfo = FunctionInfos[FunctionSymbol]; + FunctionInfo.BasicBlockInfos.insert(std::make_pair(BlockID, Info)); +} + +MCSymbol *MIPSectionEmitter::getRawProfileSymbol() { + // TODO: Take a function symbol as input so we know we are getting the right + // raw symbol. For now we are using the function we are currently in. + auto *FunctionSymbol = AP.getSymbol(&AP.MF->getFunction()); + auto &Info = FunctionInfos[FunctionSymbol]; + return Info.RawProfileSymbol; +} + +uint64_t MIPSectionEmitter::getOffsetToRawBlockProfileSymbol(uint32_t BlockID) { + assert(MIRInstrumentation::EnableMachineBasicBlockCoverage); + if (MIRInstrumentation::EnableMachineFunctionCoverage) + return 1 + BlockID; + if (MIRInstrumentation::EnableMachineCallGraph) + return 8 + BlockID; + llvm_unreachable("Expected function coverage or call graph instrumentation."); +} + +void MIPSectionEmitter::emitMIPHeader(MIPFileType FileType) { + auto &OS = *AP.OutStreamer; + + AP.emitAlignment(Align(8)); + + auto *ReferenceLabel = AP.createTempSymbol("ref"); + OS.emitLabel(ReferenceLabel); + + OS.AddComment("Magic"); + OS.emitIntValueInHex(MIP_MAGIC_VALUE, 4); + + OS.AddComment("Version"); + OS.emitIntValue(MIP_VERSION, 2); + + OS.AddComment("File Type"); + OS.emitIntValueInHex(FileType, 2); + + uint32_t ProfileType; + if (MIRInstrumentation::EnableMachineFunctionCoverage) { + ProfileType = MIP_PROFILE_TYPE_FUNCTION_COVERAGE; + } else if (MIRInstrumentation::EnableMachineCallGraph) { + ProfileType = MIP_PROFILE_TYPE_FUNCTION_TIMESTAMP | + MIP_PROFILE_TYPE_FUNCTION_CALL_COUNT; + } else { + llvm_unreachable( + "Expected function coverage or call graph instrumentation."); + } + if (MIRInstrumentation::EnableMachineBasicBlockCoverage) + ProfileType |= MIP_PROFILE_TYPE_BLOCK_COVERAGE; + OS.AddComment("Profile Type"); + OS.emitIntValueInHex(ProfileType, 4); + + OS.AddComment("Module Hash"); + OS.emitIntValueInHex((uint32_t)MD5Hash(MIRInstrumentation::LinkUnitName), 4); + + // Reserved + OS.emitZeros(4); + + OS.AddComment("Offset To Data"); + OS.emitIntValueInHex(sizeof(MIPHeader), 4); + + OS.AddBlankLine(); +} + +void MIPSectionEmitter::emitMIPFunctionData(const MFInfo &Info) { + auto &OS = *AP.OutStreamer; + auto &OutContext = OS.getContext(); + + if (Info.Func->hasComdat()) { + assert(AP.TM.getTargetTriple().getObjectFormat() == Triple::ELF); + // If the function is in a COMDAT section, that function's map info must + // also be in a COMDAT section so that it is deduplicated correctly. + auto ComdatName = Info.Func->getComdat()->getName(); + OS.SwitchSection((MCSection *)OutContext.getELFSection( + MIP_RAW_SECTION_NAME, ELF::SHT_PROGBITS, + ELF::SHF_WRITE | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, ComdatName, + /*IsComdat=*/true)); + } else { + OS.SwitchSection(OutContext.getObjectFileInfo()->getMIPRawSection()); + } + + if (MIRInstrumentation::EnableMachineFunctionCoverage) { + AP.emitAlignment(Align(1)); + AP.emitLinkage(Info.Func, Info.RawProfileSymbol); + OS.emitLabel(Info.RawProfileSymbol); + + OS.emitIntValueInHex(0xFF, 1); + } else if (MIRInstrumentation::EnableMachineCallGraph) { + AP.emitAlignment(Align(4)); + AP.emitLinkage(Info.Func, Info.RawProfileSymbol); + OS.emitLabel(Info.RawProfileSymbol); + + OS.emitIntValueInHex(0xFFFFFFFF, 4); + OS.emitIntValueInHex(0xFFFFFFFF, 4); + } else { + llvm_unreachable( + "Expected function coverage or call graph instrumentation."); + } + + if (MIRInstrumentation::EnableMachineBasicBlockCoverage) { + OS.emitFill(Info.NonEntryBasicBlockCount, 0xFF); + } + + OS.AddBlankLine(); +} + +void MIPSectionEmitter::emitMIPFunctionInfo(MFInfo &Info) { + auto &OS = *AP.OutStreamer; + auto &OutContext = OS.getContext(); + + if (Info.Func->hasComdat()) { + assert(AP.TM.getTargetTriple().getObjectFormat() == Triple::ELF); + // If the function is in a COMDAT section, that function's map info must + // also be in a COMDAT section so that it is deduplicated correctly. + auto ComdatName = Info.Func->getComdat()->getName(); + OS.SwitchSection((MCSection *)OutContext.getELFSection( + MIP_MAP_SECTION_NAME, ELF::SHT_PROGBITS, + ELF::SHF_WRITE | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, ComdatName, + /*IsComdat=*/true)); + } else { + OS.SwitchSection(OutContext.getObjectFileInfo()->getMIPMapSection()); + } + + auto MangledName = getMangledName(Info.Func); + auto *MapEntrySymbol = OutContext.getOrCreateSymbol(MangledName + "$MAP"); + AP.emitLinkage(Info.Func, MapEntrySymbol); + AP.emitAlignment(Align(8)); + OS.emitLabel(MapEntrySymbol); + + // NOTE: Since we cannot compute a difference across sections, we use two + // PC-relative relocations to represent the section-relative address of + // the `Info.RawProfileSymbol` symbol in the raw section. The actual + // section-relative address is computed by + // -
+ auto *ReferenceLabel = AP.createTempSymbol("ref"); + OS.emitLabel(ReferenceLabel); + OS.AddComment("Raw Section Start PC Offset"); + OS.emitValue( + MCBinaryExpr::createSub( + MCSymbolRefExpr::create( + getMIPSectionBeginSymbol(MIP_RAW_SECTION_NAME), OutContext), + MCSymbolRefExpr::create(ReferenceLabel, OutContext), OutContext), + 4); + OS.AddComment("Raw Profile Symbol PC Offset"); + OS.emitValue(MCBinaryExpr::createSub( + MCSymbolRefExpr::create(Info.RawProfileSymbol, OutContext), + MCSymbolRefExpr::create(ReferenceLabel, OutContext), + OutContext), + 4); + // NOTE: We use the same method to encode the offset of the function to the + // raw section. Then we can compute the absolute address of the function + // by adding the absolute address of the raw section. + OS.AddComment("Function PC Offset"); + OS.emitValue(MCBinaryExpr::createSub( + MCSymbolRefExpr::create(Info.StartSymbol, OutContext), + MCSymbolRefExpr::create(ReferenceLabel, OutContext), + OutContext), + 4); + + OS.AddComment("Function Size"); + OS.emitValue(MCBinaryExpr::createSub( + MCSymbolRefExpr::create(Info.EndSymbol, OutContext), + MCSymbolRefExpr::create(Info.StartSymbol, OutContext), + OutContext), + 4); + + OS.AddComment("CFG Signature"); + OS.emitIntValueInHex(Info.ControlFlowGraphSignature, 4); + + OS.AddComment("Non-entry Block Count"); + OS.emitIntValue(Info.NonEntryBasicBlockCount, 4); + + for (uint64_t BlockID = 0; BlockID < Info.NonEntryBasicBlockCount; + BlockID++) { + if (Info.BasicBlockInfos.count(BlockID)) { + const auto &MBBInfo = Info.BasicBlockInfos[BlockID]; + OS.AddComment("Block " + Twine(BlockID) + " Offset"); + OS.emitValue(MCBinaryExpr::createSub( + MCSymbolRefExpr::create(MBBInfo.StartSymbol, OutContext), + MCSymbolRefExpr::create(Info.StartSymbol, OutContext), + OutContext), + 4); + } else { + OS.emitZeros(4); + } + } + + OS.AddComment("Function Name Length"); + OS.emitIntValue(MangledName.size(), 4); + OS.emitBytes(MangledName); + + OS.AddBlankLine(); +} + +void MIPSectionEmitter::serializeToMIPRawSection() { + if (!MIRInstrumentation::EnableMachineInstrumentation) + return; + + auto &OS = *AP.OutStreamer; + auto &OutContext = OS.getContext(); + + // NOTE: We either emit a COMDAT section or a weak definition to ensure the + // header symbol is deduplicated correctly. + if (auto *MIPRawHeaderComdatSection = + OutContext.getObjectFileInfo()->getMIPRawHeaderComdatSection()) { + OS.SwitchSection(MIPRawHeaderComdatSection); + } else { + OS.SwitchSection(OutContext.getObjectFileInfo()->getMIPRawSection()); + auto *HeaderSymbol = getMIPSectionBeginSymbol(MIP_RAW_SECTION_NAME); + OS.emitSymbolAttribute(HeaderSymbol, MCSymbolAttr::MCSA_Global); + OS.emitSymbolAttribute(HeaderSymbol, MCSymbolAttr::MCSA_WeakDefinition); + OS.emitLabel(HeaderSymbol); + } + + emitMIPHeader(MIP_FILE_TYPE_RAW); + + for (auto &Pair : FunctionInfos) { + emitMIPFunctionData(Pair.second); + } +} + +void MIPSectionEmitter::serializeToMIPMapSection() { + if (!MIRInstrumentation::EnableMachineInstrumentation) + return; + + auto &OS = *AP.OutStreamer; + auto &OutContext = OS.getContext(); + + // NOTE: We either emit a COMDAT section or a weak definition to ensure the + // header symbol is deduplicated correctly. + if (auto *MIPMapHeaderComdatSection = + OutContext.getObjectFileInfo()->getMIPMapHeaderComdatSection()) { + OS.SwitchSection(MIPMapHeaderComdatSection); + } else { + OS.SwitchSection(OutContext.getObjectFileInfo()->getMIPMapSection()); + auto *HeaderSymbol = getMIPSectionBeginSymbol(MIP_MAP_SECTION_NAME); + OS.emitSymbolAttribute(HeaderSymbol, MCSymbolAttr::MCSA_Global); + OS.emitSymbolAttribute(HeaderSymbol, MCSymbolAttr::MCSA_WeakDefinition); + OS.emitLabel(HeaderSymbol); + } + + emitMIPHeader(MIP_FILE_TYPE_MAP); + + for (auto &Pair : FunctionInfos) { + emitMIPFunctionInfo(Pair.second); + } +} diff --git a/llvm/lib/CodeGen/MIRInstrumentationPass.cpp b/llvm/lib/CodeGen/MIRInstrumentationPass.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/MIRInstrumentationPass.cpp @@ -0,0 +1,208 @@ +//===-------------- MIRInstrumentationPass.cpp ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "machine-ir-instrumentation" + +#include "llvm/CodeGen/MIRInstrumentationPass.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; + +STATISTIC(NumInstrumented, "Number of machine functions instrumented"); +STATISTIC(NumBlocksInstrumented, "Number of machine basic blocks instrumented"); + +char MIRInstrumentation::ID; +char &llvm::MIRInstrumentationID = MIRInstrumentation::ID; +INITIALIZE_PASS(MIRInstrumentation, DEBUG_TYPE, + "Add instrumentation code to machine functions.", false, false) + +cl::opt MIRInstrumentation::EnableMachineInstrumentation( + "enable-machine-instrumentation", cl::init(false), cl::ZeroOrMore, + cl::desc("Instrument machine ir")); +cl::opt MIRInstrumentation::EnableMachineFunctionCoverage( + "enable-machine-function-coverage", cl::init(false), cl::ZeroOrMore, + cl::desc("Instrument machine ir to profile function coverage only.")); +cl::opt MIRInstrumentation::EnableMachineBasicBlockCoverage( + "enable-machine-block-coverage", cl::init(false), cl::ZeroOrMore, + cl::desc("Instrument machine ir to profile machine basic blocks.")); +cl::opt MIRInstrumentation::EnableMachineCallGraph( + "enable-machine-call-graph", cl::init(false), cl::ZeroOrMore, + cl::desc("Instrument machine ir to profile the dynamic call graph.")); +cl::opt MIRInstrumentation::MachineProfileRuntimeBufferSize( + "machine-profile-runtime-buffer", cl::init(0), cl::ZeroOrMore, + cl::value_desc("RuntimeBufferSize"), + cl::desc("Allocate a buffer of bytes to hold machine " + "function call samples.")); +cl::opt MIRInstrumentation::MachineProfileFunctionGroupCount( + "machine-profile-function-group-count", cl::init(1), cl::ZeroOrMore, + cl::value_desc("N"), + cl::desc( + "Partition the machine functions into groups and instrument the " + "group specified by -machine-profile-selected-function-group.")); +cl::opt MIRInstrumentation::MachineProfileSelectedFunctionGroup( + "machine-profile-selected-function-group", cl::init(0), cl::ZeroOrMore, + cl::value_desc("i"), + cl::desc("Instrument group . Must be in the range [0, " + "-fmachine-profile-function-group-count).")); +cl::opt MIRInstrumentation::MachineProfileMinInstructionSize( + "machine-profile-min-instruction-size", cl::init(0), cl::ZeroOrMore, + cl::value_desc("N"), + cl::desc("Do not instrument machine function that have fewer than " + "machine instructions.")); + +std::string MIRInstrumentation::LinkUnitName; +cl::opt MIRInstrumentation::LinkUnitNameOption( + "link-unit-name", cl::location(MIRInstrumentation::LinkUnitName), + cl::init(""), cl::ZeroOrMore, cl::value_desc("LinkUnitName"), + cl::desc("Use to identify this link unit")); + +bool MIRInstrumentation::doInitialization(Module &M) { + auto &Ctx = M.getContext(); + if (EnableMachineInstrumentation) { + if (EnableMachineFunctionCoverage == EnableMachineCallGraph) + Ctx.emitError("Exactly one of -" + Twine(EnableMachineCallGraph.ArgStr) + + " or -" + Twine(EnableMachineFunctionCoverage.ArgStr) + + " must be provided when using -" + + Twine(EnableMachineInstrumentation.ArgStr) + "."); + + if (EnableMachineFunctionCoverage && MachineProfileRuntimeBufferSize) + Ctx.emitError("Cannot set -" + + Twine(MachineProfileRuntimeBufferSize.ArgStr) + " when -" + + Twine(EnableMachineFunctionCoverage.ArgStr) + + " is provided."); + + if (EnableMachineCallGraph) + Ctx.emitError("-" + Twine(EnableMachineCallGraph.ArgStr) + + " is not yet implemented."); + + if (MachineProfileRuntimeBufferSize) + Ctx.emitError("-" + Twine(MachineProfileRuntimeBufferSize.ArgStr) + + " is not yet implemented."); + } + return false; +} + +bool MIRInstrumentation::runOnMachineFunction(MachineFunction &MF) { + if (!shouldInstrumentMachineFunction(MF)) + return false; + + LLVM_DEBUG(dbgs() << "Visit " << MF.getName()); + + SmallVector MBBs; + getMachineBasicBlocks(MF, MBBs); + if (MBBs.empty()) { + LLVM_DEBUG(dbgs() << MF.getName() << " has zero non-debug blocks"); + return false; + } + + auto &EntryBlock = *MBBs[0]; + unsigned NonEntryBlockCount = + EnableMachineBasicBlockCoverage ? MBBs.size() - 1 : 0; + + const auto &TII = *MF.getSubtarget().getInstrInfo(); + auto MBBI = EntryBlock.begin(); + const auto &DL = MBBI->getDebugLoc(); + BuildMI(EntryBlock, MBBI, DL, + TII.get(TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER)) + .addImm(getControlFlowGraphSignature(MBBs)) + .addImm(NonEntryBlockCount); + ++NumInstrumented; + + if (EnableMachineFunctionCoverage) { + BuildMI(EntryBlock, MBBI, DL, + TII.get(TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION)) + .addReg(TII.getTemporaryMachineProfileRegister(EntryBlock)); + } else if (EnableMachineCallGraph) { + llvm_unreachable("Not yet implemented"); + } else { + llvm_unreachable( + "Expected function coverage or call graph instrumentation."); + } + + for (uint32_t BlockID = 0; BlockID < NonEntryBlockCount; BlockID++) { + auto &MBB = *MBBs[/*EntryBlock=*/1 + BlockID]; + runOnMachineBasicBlock(MBB, BlockID); + } + + return true; +} + +void MIRInstrumentation::runOnMachineBasicBlock(MachineBasicBlock &MBB, + uint32_t BlockID) { + const auto &MF = *MBB.getParent(); + const auto &TII = *MF.getSubtarget().getInstrInfo(); + auto MBBI = MBB.begin(); + const auto &DL = MBBI->getDebugLoc(); + BuildMI(MBB, MBBI, DL, + TII.get(TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION)) + .addReg(TII.getTemporaryMachineProfileRegister(MBB)) + .addImm(BlockID); + ++NumBlocksInstrumented; +} + +bool MIRInstrumentation::shouldInstrumentMachineFunction( + const MachineFunction &MF) const { + auto Name = MF.getName(); + if (MF.empty() || Name.empty()) + return false; + + if (Name.startswith("OUTLINED_FUNCTION_")) + return false; + + if (MF.getFunction().hasFnAttribute(Attribute::Naked)) + return false; + + if (MF.getInstructionCount() < MachineProfileMinInstructionSize) + return false; + + if (MachineProfileFunctionGroupCount > 1) { + unsigned Group = MD5Hash(Name) % MachineProfileFunctionGroupCount; + if (Group != MachineProfileSelectedFunctionGroup) + return false; + } + + return true; +} + +void MIRInstrumentation::getMachineBasicBlocks( + MachineFunction &MF, SmallVectorImpl &MBBs) const { + auto ShouldSkipBlock = [](const MachineBasicBlock &MBB) { + return MBB.empty() || MBB.getFirstNonDebugInstr() == MBB.end(); + }; + + for (auto &MBB : MF) { + if (!ShouldSkipBlock(MBB)) + MBBs.push_back(&MBB); + } +} + +uint32_t MIRInstrumentation::getControlFlowGraphSignature( + SmallVectorImpl &MBBs) const { + if (MBBs.size() <= 1) + return 0; + + DenseMap BlockToID; + uint32_t ID = 0; + for (auto *MBB : MBBs) + BlockToID[MBB] = ID++; + + std::string AdjacencyList; + raw_string_ostream OS(AdjacencyList); + for (auto *MBB : MBBs) { + OS << "{"; + for (auto *Succ : MBB->successors()) + OS << BlockToID[Succ] << ";"; + OS << "}"; + } + + return MD5Hash(OS.str()); +} diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -23,6 +23,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CSEConfigBase.h" +#include "llvm/CodeGen/MIRInstrumentationPass.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachinePassRegistry.h" #include "llvm/CodeGen/Passes.h" @@ -1184,6 +1185,12 @@ addPass(createMIRAddFSDiscriminatorsPass( sampleprof::FSDiscriminatorPass::PassLast)); + // Inject machine instrumentation code into machine functions. + // NOTE: Block instrumentation may increase block size. Inject code before + // branch relaxation. + if (MIRInstrumentation::EnableMachineInstrumentation) + addPass(&MIRInstrumentationID); + addPreEmitPass(); if (TM->Options.EnableIPRA) diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp --- a/llvm/lib/MC/MCMachOStreamer.cpp +++ b/llvm/lib/MC/MCMachOStreamer.cpp @@ -30,6 +30,7 @@ #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCSymbolMachO.h" #include "llvm/MC/MCValue.h" +#include "llvm/MIP/MIP.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" @@ -143,6 +144,10 @@ SecName == "__thread_ptr")) return true; + if (SegName == "__DATA" && + (SecName == MIP_MAP_SECTION_NAME || SecName == MIP_RAW_SECTION_NAME)) + return true; + return false; } diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -20,6 +20,7 @@ #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCSectionWasm.h" #include "llvm/MC/MCSectionXCOFF.h" +#include "llvm/MIP/MIP.h" using namespace llvm; @@ -295,6 +296,13 @@ FaultMapSection = Ctx->getMachOSection("__LLVM_FAULTMAPS", "__llvm_faultmaps", 0, SectionKind::getMetadata()); + MIPRawSection = Ctx->getMachOSection("__DATA", MIP_RAW_SECTION_NAME, + MachO::S_ATTR_NO_DEAD_STRIP, 0, + SectionKind::getMetadata()); + MIPMapSection = Ctx->getMachOSection("__DATA", MIP_MAP_SECTION_NAME, + MachO::S_ATTR_NO_DEAD_STRIP, 0, + SectionKind::getMetadata()); + RemarksSection = Ctx->getMachOSection( "__LLVM", "__remarks", MachO::S_ATTR_DEBUG, SectionKind::getMetadata()); @@ -494,6 +502,19 @@ FaultMapSection = Ctx->getELFSection(".llvm_faultmaps", ELF::SHT_PROGBITS, ELF::SHF_ALLOC); + MIPRawSection = Ctx->getELFSection(MIP_RAW_SECTION_NAME, ELF::SHT_PROGBITS, + ELF::SHF_WRITE | ELF::SHF_ALLOC); + MIPMapSection = Ctx->getELFSection(MIP_MAP_SECTION_NAME, ELF::SHT_PROGBITS, + ELF::SHF_WRITE | ELF::SHF_ALLOC); + MIPRawHeaderComdatSection = + Ctx->getELFSection(MIP_RAW_SECTION_NAME, ELF::SHT_PROGBITS, + ELF::SHF_WRITE | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + "Header", /*IsComdat=*/true); + MIPMapHeaderComdatSection = + Ctx->getELFSection(MIP_MAP_SECTION_NAME, ELF::SHT_PROGBITS, + ELF::SHF_WRITE | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + "Header", /*IsComdat=*/true); + EHFrameSection = Ctx->getELFSection(".eh_frame", EHSectionType, EHSectionFlags); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -103,6 +103,9 @@ void LowerFAULTING_OP(const MachineInstr &MI); void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); + void LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(const MachineInstr &MI); + void LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(const MachineInstr &MI); + void LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(const MachineInstr &MI); void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); @@ -247,6 +250,96 @@ OutStreamer->GetCommentOS() << ' ' << OutlinerString; } +void AArch64AsmPrinter::LowerMIP_FUNCTION_INSTRUMENTATION_MARKER( + const MachineInstr &MI) { + MIPEmitter.runOnFunctionInstrumentationMarker(MI); +} + +void AArch64AsmPrinter::LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION( + const MachineInstr &MI) { + auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(); + + const auto ProfileRegister = MI.getOperand(0).getReg(); + auto RawAddressPageMO = + MachineOperand::CreateMCSymbol(RawProfileSymbol, AArch64II::MO_PAGE); + auto RawAddressPageOffsetMO = MachineOperand::CreateMCSymbol( + RawProfileSymbol, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + MCOperand RawAddressPageMCO, RawAddressPageOffsetMCO; + lowerOperand(RawAddressPageMO, RawAddressPageMCO); + lowerOperand(RawAddressPageOffsetMO, RawAddressPageOffsetMCO); + + OutStreamer->AddComment("MIP: Function Coverage"); + // adrp , + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADRP) + .addReg(ProfileRegister) + .addOperand(RawAddressPageMCO)); + // strb wzr, [, ] + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::STRBBui) + .addReg(AArch64::WZR) + .addReg(ProfileRegister) + .addOperand(RawAddressPageOffsetMCO)); +} + +void AArch64AsmPrinter::LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION( + const MachineInstr &MI) { + MIPEmitter.runOnBasicBlockInstrumentationMarker(MI); + + auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(); + auto TempRegister = MI.getOperand(0).getReg(); + auto BlockID = MI.getOperand(1).getImm(); + auto Offset = MIPEmitter.getOffsetToRawBlockProfileSymbol(BlockID); + bool ShouldSpillRegister = TempRegister == AArch64::NoRegister; + + auto RawAddressPageMO = + MachineOperand::CreateMCSymbol(RawProfileSymbol, AArch64II::MO_PAGE); + auto RawAddressPageOffsetMO = MachineOperand::CreateMCSymbol( + RawProfileSymbol, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + MCOperand RawAddressPageMCO, RawAddressPageOffsetMCO; + lowerOperand(RawAddressPageMO, RawAddressPageMCO); + lowerOperand(RawAddressPageOffsetMO, RawAddressPageOffsetMCO); + + OutStreamer->AddComment("MIP: Block Coverage"); + // TODO: There is still some oportunity for optimization here. If there is a + // register that is dead for the whole function, we can use it to store + // the raw profile address at the beginning of the function. Then for + // each block we use one `strb` instruction with an offset if the offset + // is less than 4096. + if (ShouldSpillRegister) { + TempRegister = AArch64::X16; + // Spill the temporary register. + // str , [sp, #-16]! + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::STRXpre) + .addReg(AArch64::SP) + .addReg(TempRegister) + .addReg(AArch64::SP) + .addImm(-16)); + } + // adrp , + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADRP) + .addReg(TempRegister) + .addOperand(RawAddressPageMCO)); + // add , , + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri) + .addReg(TempRegister) + .addReg(TempRegister) + .addImm(Offset) + .addImm(0)); + // strb wzr, [, ] + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::STRBBui) + .addReg(AArch64::WZR) + .addReg(TempRegister) + .addOperand(RawAddressPageOffsetMCO)); + if (ShouldSpillRegister) { + // Restore the temporary register. + // ldr , [sp], #16 + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXpost) + .addReg(AArch64::SP) + .addReg(TempRegister) + .addReg(AArch64::SP) + .addImm(16)); + } +} + void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) { const Function &F = MF->getFunction(); @@ -1357,6 +1450,15 @@ case TargetOpcode::STACKMAP: return LowerSTACKMAP(*OutStreamer, SM, *MI); + case TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER: + return LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(*MI); + + case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION: + return LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(*MI); + + case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION: + return LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(*MI); + case TargetOpcode::PATCHPOINT: return LowerPATCHPOINT(*OutStreamer, SM, *MI); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -319,6 +319,9 @@ static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized); + Register getTemporaryMachineProfileRegister( + const MachineBasicBlock &MBB) const override; + #define GET_INSTRINFO_HELPER_DECLS #include "AArch64GenInstrInfo.inc" diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -98,6 +98,12 @@ // Anything not explicitly designated otherwise is a normal 4-byte insn. NumBytes = 4; break; + case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION: + NumBytes = 8; + break; + case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION: + NumBytes = (MI.getOperand(0).getReg() == AArch64::NoRegister) ? 20 : 12; + break; case TargetOpcode::STACKMAP: // The upper bound for a stackmap intrinsic is the full length of its shadow NumBytes = StackMapOpers(&MI).getNumPatchBytes(); @@ -3930,6 +3936,23 @@ } } +Register AArch64InstrInfo::getTemporaryMachineProfileRegister( + const MachineBasicBlock &MBB) const { + const auto &MF = *MBB.getParent(); + const auto &TRI = getRegisterInfo(); + if (MF.getRegInfo().tracksLiveness()) { + LiveRegUnits LRU(TRI); + LRU.addReg(AArch64::LR); + LRU.addUnits(TRI.getReservedRegs(MF)); + LRU.addLiveIns(MBB); + for (unsigned Reg : AArch64::GPR64RegClass) { + if (LRU.available(Reg)) + return Reg; + } + } + return AArch64::NoRegister; +} + // Helper function to emit a frame offset adjustment from a given // pointer (SrcReg), stored into DestReg. This function is explicit // in that it requires the opcode. diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.h b/llvm/lib/Target/ARM/ARMAsmPrinter.h --- a/llvm/lib/Target/ARM/ARMAsmPrinter.h +++ b/llvm/lib/Target/ARM/ARMAsmPrinter.h @@ -114,6 +114,11 @@ void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); + // MIP-specific lowering for ARM. + void LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(const MachineInstr &MI); + void LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(const MachineInstr &MI); + void LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(const MachineInstr &MI); + private: void EmitSled(const MachineInstr &MI, SledKind Kind); diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp --- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp +++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp @@ -2180,6 +2180,12 @@ case ARM::PATCHABLE_TAIL_CALL: LowerPATCHABLE_TAIL_CALL(*MI); return; + case TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER: + return LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(*MI); + case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION: + return LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(*MI); + case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION: + return LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(*MI); case ARM::SpeculationBarrierISBDSBEndBB: { // Print DSB SYS + ISB MCInst TmpInstDSB; diff --git a/llvm/lib/Target/ARM/ARMMCInstLower.cpp b/llvm/lib/Target/ARM/ARMMCInstLower.cpp --- a/llvm/lib/Target/ARM/ARMMCInstLower.cpp +++ b/llvm/lib/Target/ARM/ARMMCInstLower.cpp @@ -227,3 +227,152 @@ { EmitSled(MI, SledKind::TAIL_CALL); } + +void ARMAsmPrinter::LowerMIP_FUNCTION_INSTRUMENTATION_MARKER( + const MachineInstr &MI) { + MIPEmitter.runOnFunctionInstrumentationMarker(MI); +} + +void ARMAsmPrinter::LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION( + const MachineInstr &MI) { + auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(); + + auto &AFI = *MI.getParent()->getParent()->getInfo(); + auto *RawProfileSymbolLocationLabel = + OutContext.createTempSymbol("RawSymbolLoc", true); + auto *LoadLabel = OutContext.createTempSymbol("LoadLabel", true); + auto *ContinueLabel = OutContext.createTempSymbol("ContinueLabel", true); + const MCExpr *RawProfileSymbolLocation = + MCSymbolRefExpr::create(RawProfileSymbolLocationLabel, OutContext); + + // TODO: The emitted code is very unoptimized, but we initially strive for + // correctness. + OutStreamer->AddComment("MIP: Function Coverage"); + // push {r0, r1} + auto PushOpcode = AFI.isThumbFunction() ? ARM::tPUSH : ARM::STMDB_UPD; + auto PushInst = MCInstBuilder(PushOpcode); + if (!AFI.isThumbFunction()) + PushInst.addReg(ARM::SP).addReg(ARM::SP); + PushInst.addImm(ARMCC::AL) + .addReg(ARM::NoRegister) + .addReg(ARM::R0) + .addReg(ARM::R1); + EmitToStreamer(*OutStreamer, PushInst); + + // ldr r1, + if (AFI.isThumbFunction()) { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tLDRpci) + .addReg(ARM::R1) + .addOperand(MCOperand::createExpr( + RawProfileSymbolLocation)) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + } else { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::LDRi12) + .addReg(ARM::R1) + .addOperand(MCOperand::createExpr( + RawProfileSymbolLocation)) + .addImm(0) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + } + + // : + // add r1, pc, r1 + OutStreamer->emitLabel(LoadLabel); + if (AFI.isThumbFunction()) { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tADDhirr) + .addReg(ARM::R1) + .addReg(ARM::R1) + .addReg(ARM::PC) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + } else { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::ADDrr) + .addReg(ARM::R1) + .addReg(ARM::PC) + .addReg(ARM::R1) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister) + .addReg(ARM::NoRegister)); + } + + // mov r0, #0 + if (AFI.isThumbFunction()) { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tMOVi8) + .addReg(ARM::R0) + .addReg(ARM::CPSR) + .addImm(0) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + } else { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::MOVi) + .addReg(ARM::R0) + .addImm(0) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister) + .addReg(ARM::NoRegister)); + } + + // strb r0, [r1] + if (AFI.isThumbFunction()) { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tSTRBi) + .addReg(ARM::R0) + .addReg(ARM::R1) + .addImm(0) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + } else { + EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::STRB_POST_IMM) + .addReg(ARM::R1) + .addReg(ARM::R0) + .addReg(ARM::R1) + .addReg(ARM::NoRegister) + .addImm(0) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + } + + // pop {r0, r1} + auto PopOpcode = AFI.isThumbFunction() ? ARM::tPOP : ARM::LDMIA_UPD; + auto PopInst = MCInstBuilder(PopOpcode); + if (!AFI.isThumbFunction()) + PopInst.addReg(ARM::SP).addReg(ARM::SP); + PopInst.addImm(ARMCC::AL) + .addReg(ARM::NoRegister) + .addReg(ARM::R0) + .addReg(ARM::R1); + EmitToStreamer(*OutStreamer, PopInst); + + // b + auto BranchOpcode = AFI.isThumbFunction() ? ARM::tB : ARM::Bcc; + EmitToStreamer(*OutStreamer, MCInstBuilder(BranchOpcode) + .addExpr(MCSymbolRefExpr::create( + ContinueLabel, OutContext)) + .addImm(ARMCC::AL) + .addReg(ARM::NoRegister)); + // NOTE: T16 LDR instructions require labels to be 4 byte aligned. + // .p2align 2 + // : + // .long -(+) + OutStreamer->emitCodeAlignment(4); + OutStreamer->emitLabel(RawProfileSymbolLocationLabel); + // NOTE: In ARM the value of PC is the address of the current instruction + // plus 8 bytes, but in Thumb it's 4 bytes. + int64_t PCFixupValue = AFI.isThumbFunction() ? 4 : 8; + const auto *PCRelativeAddress = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(RawProfileSymbol, OutContext), + MCBinaryExpr::createAdd(MCSymbolRefExpr::create(LoadLabel, OutContext), + MCConstantExpr::create(PCFixupValue, OutContext), + OutContext), + OutContext); + OutStreamer->emitValue(PCRelativeAddress, 4); + + // : + OutStreamer->emitLabel(ContinueLabel); +} + +void ARMAsmPrinter::LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION( + const MachineInstr &MI) { + llvm_unreachable("MIP block coverage is not implemlented for ARM targets"); +} diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h --- a/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/llvm/lib/Target/X86/X86AsmPrinter.h @@ -87,6 +87,10 @@ void LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI); + void LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(const MachineInstr &MI); + void LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(const MachineInstr &MI); + void LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(const MachineInstr &MI); + // XRay-specific lowering for X86. void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, X86MCInstLower &MCIL); diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1385,6 +1385,52 @@ SMShadowTracker.reset(NumShadowBytes); } +void X86AsmPrinter::LowerMIP_FUNCTION_INSTRUMENTATION_MARKER( + const MachineInstr &MI) { + MIPEmitter.runOnFunctionInstrumentationMarker(MI); +} + +void X86AsmPrinter::LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION( + const MachineInstr &MI) { + auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(); + + OutStreamer->AddComment("MIP: Function Coverage"); + // movb $0, (%rsp) + EmitAndCountInstruction( + MCInstBuilder(X86::MOV8mi) + .addReg(X86::RIP) // BaseReg + .addImm(1) // ScaleAmt + .addReg(0) // IndexReg + .addExpr(MCSymbolRefExpr::create(RawProfileSymbol, OutContext)) + .addReg(0) // Segment + .addImm(0) // Immediate + ); +} + +void X86AsmPrinter::LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION( + const MachineInstr &MI) { + MIPEmitter.runOnBasicBlockInstrumentationMarker(MI); + + auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(); + + auto BlockID = MI.getOperand(1).getImm(); + auto Offset = MIPEmitter.getOffsetToRawBlockProfileSymbol(BlockID); + + OutStreamer->AddComment("MIP: Block Coverage"); + // movb $0, +(%rsp) + EmitAndCountInstruction( + MCInstBuilder(X86::MOV8mi) + .addReg(X86::RIP) // BaseReg + .addImm(1) // ScaleAmt + .addReg(0) // IndexReg + .addExpr(MCBinaryExpr::createAdd( + MCSymbolRefExpr::create(RawProfileSymbol, OutContext), + MCConstantExpr::create(Offset, OutContext), OutContext)) + .addReg(0) // Segment + .addImm(0) // Immediate + ); +} + // Lower a patchpoint of the form: // [], , , , , , ... void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, @@ -2541,6 +2587,15 @@ case TargetOpcode::STACKMAP: return LowerSTACKMAP(*MI); + case TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER: + return LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(*MI); + + case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION: + return LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(*MI); + + case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION: + return LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(*MI); + case TargetOpcode::PATCHPOINT: return LowerPATCHPOINT(*MI, MCInstLowering); diff --git a/llvm/test/CodeGen/AArch64/mip-basic-block-coverage.ll b/llvm/test/CodeGen/AArch64/mip-basic-block-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/mip-basic-block-coverage.ll @@ -0,0 +1,66 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=arm64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=arm64-apple-ios | FileCheck %s --check-prefixes CHECK,CHECK-MACHO + +; CHECK-ELF-LABEL: _Z3fooi: +; CHECK-MACHO-LABEL: __Z3fooi: +define i32 @_Z3fooi(i32 %a) { +entry: + %a.addr = alloca i32, align 4 + %sum = alloca i32, align 4 + %i = alloca i32, align 4 + store i32 %a, i32* %a.addr, align 4 + store i32 0, i32* %sum, align 4 + store i32 0, i32* %i, align 4 + br label %for.cond + +; CHECK-LABEL: LBB0_1: +for.cond: ; preds = %for.inc, %entry + ; CHECK-ELF: adrp [[r:x[0-9]+]], _Z3fooi$RAW + ; CHECK-MACHO: adrp [[r:x[0-9]+]], __Z3fooi$RAW@PAGE + ; CHECK-NEXT: add [[r]], [[r]], #1 + ; CHECK-ELF-NEXT: strb wzr, {{\[}}[[r]], :lo12:_Z3fooi$RAW] + ; CHECK-MACHO-NEXT: strb wzr, {{\[}}[[r]], __Z3fooi$RAW@PAGEOFF] + %0 = load i32, i32* %i, align 4 + %1 = load i32, i32* %a.addr, align 4 + %cmp = icmp slt i32 %0, %1 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + ; CHECK-ELF: adrp [[r:x[0-9]+]], _Z3fooi$RAW + ; CHECK-MACHO: adrp [[r:x[0-9]+]], __Z3fooi$RAW@PAGE + ; CHECK-NEXT: add [[r]], [[r]], #2 + ; CHECK-ELF-NEXT: strb wzr, {{\[}}[[r]], :lo12:_Z3fooi$RAW] + ; CHECK-MACHO-NEXT: strb wzr, {{\[}}[[r]], __Z3fooi$RAW@PAGEOFF] + %2 = load i32, i32* %i, align 4 + %3 = load i32, i32* %sum, align 4 + %add = add nsw i32 %3, %2 + store i32 %add, i32* %sum, align 4 + br label %for.inc + +for.inc: ; preds = %for.body + ; CHECK-ELF: adrp [[r:x[0-9]+]], _Z3fooi$RAW + ; CHECK-MACHO: adrp [[r:x[0-9]+]], __Z3fooi$RAW@PAGE + ; CHECK-NEXT: add [[r]], [[r]], #3 + ; CHECK-ELF-NEXT: strb wzr, {{\[}}[[r]], :lo12:_Z3fooi$RAW] + ; CHECK-MACHO-NEXT: strb wzr, {{\[}}[[r]], __Z3fooi$RAW@PAGEOFF] + %4 = load i32, i32* %i, align 4 + %inc = add nsw i32 %4, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + %5 = load i32, i32* %sum, align 4 + ret i32 %5 + ; CHECK: ret +} + +; CHECK-ELF-LABEL: .section __llvm_mipraw +; CHECK-MACHO-LABEL: .section __DATA,__llvm_mipraw + +; CHECK-ELF-LABEL: _Z3fooi$RAW: +; CHECK-ELF-NEXT: .byte 0xff +; CHECK-ELF-NEXT: .zero 3,255 + +; CHECK-MACHO-LABEL: __Z3fooi$RAW: +; CHECK-MACHO-NEXT: .byte 0xff +; CHECK-MACHO-NEXT: .space 3,255 diff --git a/llvm/test/CodeGen/AArch64/mip-comdat.ll b/llvm/test/CodeGen/AArch64/mip-comdat.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/mip-comdat.ll @@ -0,0 +1,12 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-linux | FileCheck %s + +; CHECK-DAG: .section .text.foo,"axG",@progbits,foo,comdat +$foo = comdat any +define i32 @foo(i32) comdat { + ret i32 101 +} + + +; CHECK-DAG: .section __llvm_mipraw,"aGw",@progbits,foo,comdat + +; CHECK-DAG: .section __llvm_mipmap,"aGw",@progbits,foo,comdat diff --git a/llvm/test/CodeGen/AArch64/mip-function-coverage.ll b/llvm/test/CodeGen/AArch64/mip-function-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/mip-function-coverage.ll @@ -0,0 +1,21 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-apple-ios | FileCheck %s --check-prefixes CHECK,CHECK-MACHO + +; CHECK-ELF-LABEL: _Z3foov: +; CHECK-MACHO-LABEL: __Z3foov: +define i32 @_Z3foov() #0 { + ; CHECK-ELF: adrp [[r:x[0-9]+]], _Z3foov$RAW + ; CHECK-ELF-NEXT: strb wzr, {{\[}}[[r]], :lo12:_Z3foov$RAW] + + ; CHECK-MACHO: adrp [[r:x[0-9]+]], __Z3foov$RAW@PAGE + ; CHECK-MACHO-NEXT: strb wzr, {{\[}}[[r]], __Z3foov$RAW@PAGEOFF] + ret i32 0 + ; CHECK: ret +} + +; CHECK-ELF-LABEL: .section __llvm_mipraw +; CHECK-MACHO-LABEL: .section __DATA,__llvm_mipraw + +; CHECK-ELF-LABEL: _Z3foov$RAW: +; CHECK-MACHO-LABEL: __Z3foov$RAW: +; CHECK-NEXT: .byte 0xff diff --git a/llvm/test/CodeGen/AArch64/mip-header.ll b/llvm/test/CodeGen/AArch64/mip-header.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/mip-header.ll @@ -0,0 +1,58 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-linux | FileCheck %s --check-prefix ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-apple-ios | FileCheck %s --check-prefix MACHO + +define i32 @_Z3fooii(i32 %a, i32 %b) #0 { + ret i32 0 +} + +;================================= .mipraw Header =====================================; +; ELF: .section __llvm_mipraw,"aGw",@progbits,Header,comdat +; ELF: .p2align 3 +; ELF: [[RAW_REF:.*]]: +; ELF: .word 0x50494dfb // Magic +; ELF-NEXT: .hword 8 // Version +; ELF-NEXT: .hword 0x1 // File Type +; ELF-NEXT: .word 0x1 // Profile Type +; ELF-NEXT: .word [[MODULE_HASH:.*]] // Module Hash +; ELF-NEXT: .zero 4 +; ELF-NEXT: .word 0x18 // Offset To Data + +;================================= .mipmap Header =====================================; +; ELF: .section __llvm_mipmap,"aGw",@progbits,Header,comdat +; ELF: .p2align 3 +; ELF: [[MAP_REF:.*]]: +; ELF: .word 0x50494dfb // Magic +; ELF-NEXT: .hword 8 // Version +; ELF-NEXT: .hword 0x2 // File Type +; ELF-NEXT: .word 0x1 // Profile Type +; ELF-NEXT: .word [[MODULE_HASH]] // Module Hash +; ELF-NEXT: .zero 4 +; ELF-NEXT: .word 0x18 // Offset To Data + +;================================= .mipraw Header =====================================; +; MACHO: .section __DATA,__llvm_mipraw,regular,no_dead_strip +; MACHO: .weak_definition __header$__llvm_mipraw +; MACHO-LABEL: __header$__llvm_mipraw: +; MACHO: .p2align 3 +; MACHO: [[RAW_REF:.*]]: +; MACHO: .long 0x50494dfb ; Magic +; MACHO-NEXT: .short 8 ; Version +; MACHO-NEXT: .short 0x1 ; File Type +; MACHO-NEXT: .long 0x1 ; Profile Type +; MACHO-NEXT: .long [[MODULE_HASH:.*]] ; Module Hash +; MACHO-NEXT: .space 4 +; MACHO-NEXT: .long 0x18 ; Offset To Data + +;================================= .mipmap Header =====================================; +; MACHO: .section __DATA,__llvm_mipmap,regular,no_dead_strip +; MACHO: .weak_definition __header$__llvm_mipmap +; MACHO-LABEL: __header$__llvm_mipmap: +; MACHO: .p2align 3 +; MACHO: [[MAP_REF:.*]]: +; MACHO: .long 0x50494dfb ; Magic +; MACHO-NEXT: .short 8 ; Version +; MACHO-NEXT: .short 0x2 ; File Type +; MACHO-NEXT: .long 0x1 ; Profile Type +; MACHO-NEXT: .long [[MODULE_HASH]] ; Module Hash +; MACHO-NEXT: .space 4 +; MACHO-NEXT: .long 0x18 ; Offset To Data diff --git a/llvm/test/CodeGen/AArch64/mip-map.ll b/llvm/test/CodeGen/AArch64/mip-map.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/mip-map.ll @@ -0,0 +1,90 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=arm64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=arm64-apple-ios | FileCheck %s --check-prefixes CHECK,CHECK-MACHO + +@global = local_unnamed_addr global i32 4, align 4 + +; CHECK-LABEL: _Z3fooi: +define i32 @_Z3fooi(i32) { + ret i32 101 +} +; CHECK: [[FOO_END:.?Lmip_func_end[0-9]+]]: + +; CHECK-LABEL: _Z3gooi: +define weak i32 @_Z3gooi(i32 %a) local_unnamed_addr #0 { +entry: + %cmp = icmp sgt i32 %a, 1 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry +; CHECK: [[GOO_BLOCK0:.?LBB.*]]: + %0 = load i32, i32* @global, align 4 + %call = tail call i32 @_Z3fooi(i32 1) #2 + %add.neg = sub i32 %a, %0 + %sub = sub i32 %add.neg, %call + br label %if.end + +if.end: ; preds = %if.then, %entry +; CHECK: [[GOO_BLOCK1:.?LBB.*]]: + %c.0 = phi i32 [ %sub, %if.then ], [ %a, %entry ] + %mul = mul nsw i32 %c.0, %a + ret i32 %mul +} +; CHECK: [[GOO_END:.?Lmip_func_end[0-9]+]]: + +attributes #0 = { minsize norecurse optsize ssp uwtable "frame-pointer"="non-leaf" } + +; CHECK-ELF-LABEL: .section __llvm_mipmap,"aw",@progbits + +; CHECK-ELF: .p2align 3 +; CHECK-ELF-LABEL: _Z3fooi$MAP: +; CHECK-ELF-NEXT: [[FOO_REF:.*]]: +; CHECK-ELF-NEXT: .word __start___llvm_mipraw-[[FOO_REF]] // Raw Section Start PC Offset +; CHECK-ELF-NEXT: .word _Z3fooi$RAW-[[FOO_REF]] // Raw Profile Symbol PC Offset +; CHECK-ELF-NEXT: .word _Z3fooi-[[FOO_REF]] // Function PC Offset +; CHECK-ELF-NEXT: .word [[FOO_END]]-_Z3fooi // Function Size +; CHECK-ELF-NEXT: .word 0x0 // CFG Signature +; CHECK-ELF-NEXT: .word 0 // Non-entry Block Count +; CHECK-ELF-NEXT: .word 7 // Function Name Length +; CHECK-ELF-NEXT: .ascii "_Z3fooi" + +; CHECK-ELF: .p2align 3 +; CHECK-ELF-LABEL: _Z3gooi$MAP: +; CHECK-ELF-NEXT: [[GOO_REF:.*]]: +; CHECK-ELF-NEXT: .word __start___llvm_mipraw-[[GOO_REF]] // Raw Section Start PC Offset +; CHECK-ELF-NEXT: .word _Z3gooi$RAW-[[GOO_REF]] // Raw Profile Symbol PC Offset +; CHECK-ELF-NEXT: .word _Z3gooi-[[GOO_REF]] // Function PC Offset +; CHECK-ELF-NEXT: .word [[GOO_END]]-_Z3gooi // Function Size +; CHECK-ELF-NEXT: .word 0x70c9fa27 // CFG Signature +; CHECK-ELF-NEXT: .word 2 // Non-entry Block Count +; CHECK-ELF-NEXT: .word [[GOO_BLOCK0]]-_Z3gooi // Block 0 Offset +; CHECK-ELF-NEXT: .word [[GOO_BLOCK1]]-_Z3gooi // Block 1 Offset +; CHECK-ELF-NEXT: .word 7 // Function Name Length +; CHECK-ELF-NEXT: .ascii "_Z3gooi" + +; CHECK-MACHO-LABEL: .section __DATA,__llvm_mipmap + +; CHECK-MACHO: .p2align 3 +; CHECK-MACHO-LABEL: __Z3fooi$MAP: +; CHECK-MACHO-NEXT: [[FOO_REF:.*]]: +; CHECK-MACHO-NEXT: .long __header$__llvm_mipraw-[[FOO_REF]] ; Raw Section Start PC Offset +; CHECK-MACHO-NEXT: .long __Z3fooi$RAW-[[FOO_REF]] ; Raw Profile Symbol PC Offset +; CHECK-MACHO-NEXT: .long __Z3fooi-[[FOO_REF]] ; Function PC Offset +; CHECK-MACHO-NEXT: .long [[FOO_END]]-__Z3fooi ; Function Size +; CHECK-MACHO-NEXT: .long 0x0 ; CFG Signature +; CHECK-MACHO-NEXT: .long 0 ; Non-entry Block Count +; CHECK-MACHO-NEXT: .long 8 ; Function Name Length +; CHECK-MACHO-NEXT: .ascii "__Z3fooi" + +; CHECK-MACHO: .p2align 3 +; CHECK-MACHO-LABEL: __Z3gooi$MAP: +; CHECK-MACHO-NEXT: [[GOO_REF:.*]]: +; CHECK-MACHO-NEXT: .long __header$__llvm_mipraw-[[GOO_REF]] ; Raw Section Start PC Offset +; CHECK-MACHO-NEXT: .long __Z3gooi$RAW-[[GOO_REF]] ; Raw Profile Symbol PC Offset +; CHECK-MACHO-NEXT: .long __Z3gooi-[[GOO_REF]] ; Function PC Offset +; CHECK-MACHO-NEXT: .long [[GOO_END]]-__Z3gooi ; Function Size +; CHECK-MACHO-NEXT: .long 0x70c9fa27 ; CFG Signature +; CHECK-MACHO-NEXT: .long 2 ; Non-entry Block Count +; CHECK-MACHO-NEXT: .long [[GOO_BLOCK0]]-__Z3gooi ; Block 0 Offset +; CHECK-MACHO-NEXT: .long [[GOO_BLOCK1]]-__Z3gooi ; Block 1 Offset +; CHECK-MACHO-NEXT: .long 8 ; Function Name Length +; CHECK-MACHO-NEXT: .ascii "__Z3gooi" diff --git a/llvm/test/CodeGen/ARM/mip-comdat.ll b/llvm/test/CodeGen/ARM/mip-comdat.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/ARM/mip-comdat.ll @@ -0,0 +1,12 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=armv7-linux | FileCheck %s + +; CHECK-DAG: .section .text.foo,"axG",%progbits,foo,comdat +$foo = comdat any +define i32 @foo(i32) comdat { + ret i32 101 +} + + +; CHECK-DAG: .section __llvm_mipraw,"aGw",%progbits,foo,comdat + +; CHECK-DAG: .section __llvm_mipmap,"aGw",%progbits,foo,comdat diff --git a/llvm/test/CodeGen/ARM/mip-function-coverage.ll b/llvm/test/CodeGen/ARM/mip-function-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/ARM/mip-function-coverage.ll @@ -0,0 +1,25 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=armv7-linux | FileCheck %s + +; CHECK-LABEL: _Z3foov: +define i32 @_Z3foov() #0 { + ; CHECK: push {r0, r1} + ; CHECK-NEXT: ldr r1, [[LABEL:.*]] + ; CHECK-NEXT: [[LOAD_LABEL:.*]]: + ; CHECK-NEXT: add r1, pc, r1 + ; CHECK-NEXT: mov r0, #0 + ; CHECK-NEXT: strb r0, [r1], #0 + ; CHECK-NEXT: pop {r0, r1} + ; CHECK-NEXT: b [[CONTINUE:.*]] + ; CHECK-NEXT: .p2align 2 + ; CHECK-NEXT: [[LABEL]]: + ; CHECK-NEXT: .long _Z3foov$RAW-([[LOAD_LABEL]]+8) + ; CHECK-NEXT: [[CONTINUE]]: + + ret i32 0 + ; CHECK: bx lr +} + +; CHECK-LABEL: .section __llvm_mipraw + +; CHECK-LABEL: _Z3foov$RAW: +; CHECK-NEXT: .byte 0xff diff --git a/llvm/test/CodeGen/ARM/mip-header.ll b/llvm/test/CodeGen/ARM/mip-header.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/ARM/mip-header.ll @@ -0,0 +1,29 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=armv7-linux | FileCheck %s + +define i32 @_Z3fooii(i32 %a, i32 %b) #0 { + ret i32 0 +} + +;================================= .mipraw Header =====================================; +; CHECK: .section __llvm_mipraw,"aGw",%progbits,Header,comdat +; CHECK: .p2align 3 +; CHECK: [[RAW_REF:.*]]: +; CHECK: .long 0x50494dfb @ Magic +; CHECK-NEXT: .short 8 @ Version +; CHECK-NEXT: .short 0x1 @ File Type +; CHECK-NEXT: .long 0x1 @ Profile Type +; CHECK-NEXT: .long [[MODULE_HASH:.*]] @ Module Hash +; CHECK-NEXT: .zero 4 +; CHECK-NEXT: .long 0x18 @ Offset To Data + +;================================= .mipmap Header =====================================; +; CHECK: .section __llvm_mipmap,"aGw",%progbits,Header,comdat +; CHECK: .p2align 3 +; CHECK: [[RAW_REF:.*]]: +; CHECK: .long 0x50494dfb @ Magic +; CHECK-NEXT: .short 8 @ Version +; CHECK-NEXT: .short 0x2 @ File Type +; CHECK-NEXT: .long 0x1 @ Profile Type +; CHECK-NEXT: .long [[MODULE_HASH:.*]] @ Module Hash +; CHECK-NEXT: .zero 4 +; CHECK-NEXT: .long 0x18 @ Offset To Data diff --git a/llvm/test/CodeGen/ARM/mip-map.ll b/llvm/test/CodeGen/ARM/mip-map.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/ARM/mip-map.ll @@ -0,0 +1,57 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=armv7-linux | FileCheck %s + +@global = local_unnamed_addr global i32 4, align 4 + +; CHECK-LABEL: _Z3fooi: +define i32 @_Z3fooi(i32) { + ret i32 101 +} +; CHECK: [[FOO_END:.Lmip_func_end[0-9]+]]: + +; CHECK-LABEL: _Z3gooi: +define weak i32 @_Z3gooi(i32 %a) local_unnamed_addr #0 { +entry: + %cmp = icmp sgt i32 %a, 1 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %0 = load i32, i32* @global, align 4 + %call = tail call i32 @_Z3fooi(i32 1) #2 + %add.neg = sub i32 %a, %0 + %sub = sub i32 %add.neg, %call + br label %if.end + +if.end: ; preds = %if.then, %entry + %c.0 = phi i32 [ %sub, %if.then ], [ %a, %entry ] + %mul = mul nsw i32 %c.0, %a + ret i32 %mul +} +; CHECK: [[GOO_END:.Lmip_func_end[0-9]+]]: + +attributes #0 = { minsize norecurse optsize ssp uwtable "frame-pointer"="non-leaf" } + +; CHECK-ELF-LABEL: .section __llvm_mipmap,"aw",%progbits + +; CHECK: .p2align 3 +; CHECK-LABEL: _Z3fooi$MAP: +; CHECK-NEXT: [[FOO_REF:.*]]: +; CHECK-NEXT: .long __start___llvm_mipraw-[[FOO_REF]] @ Raw Section Start PC Offset +; CHECK-NEXT: .long _Z3fooi$RAW-[[FOO_REF]] @ Raw Profile Symbol PC Offset +; CHECK-NEXT: .long _Z3fooi-[[FOO_REF]] @ Function PC Offset +; CHECK-NEXT: .long [[FOO_END]]-_Z3fooi @ Function Size +; CHECK-NEXT: .long 0x0 @ CFG Signature +; CHECK-NEXT: .long 0 @ Non-entry Block Count +; CHECK-NEXT: .long 7 @ Function Name Length +; CHECK-NEXT: .ascii "_Z3fooi" + +; CHECK: .p2align 3 +; CHECK-LABEL: _Z3gooi$MAP: +; CHECK-NEXT: [[GOO_REF:.*]]: +; CHECK-NEXT: .long __start___llvm_mipraw-[[GOO_REF]] @ Raw Section Start PC Offset +; CHECK-NEXT: .long _Z3gooi$RAW-[[GOO_REF]] @ Raw Profile Symbol PC Offset +; CHECK-NEXT: .long _Z3gooi-[[GOO_REF]] @ Function PC Offset +; CHECK-NEXT: .long [[GOO_END]]-_Z3gooi @ Function Size +; CHECK-NEXT: .long 0x70c9fa27 @ CFG Signature +; CHECK-NEXT: .long 0 @ Non-entry Block Count +; CHECK-NEXT: .long 7 @ Function Name Length +; CHECK-NEXT: .ascii "_Z3gooi" diff --git a/llvm/test/CodeGen/Thumb/mip-function-coverage.ll b/llvm/test/CodeGen/Thumb/mip-function-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/Thumb/mip-function-coverage.ll @@ -0,0 +1,25 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=thumb-linux | FileCheck %s + +; CHECK-LABEL: _Z3foov: +define i32 @_Z3foov() #0 { + ; CHECK: push {r0, r1} + ; CHECK-NEXT: ldr r1, [[LABEL:.*]] + ; CHECK-NEXT: [[LOAD_LABEL:.*]]: + ; CHECK-NEXT: add r1, pc + ; CHECK-NEXT: movs r0, #0 + ; CHECK-NEXT: strb r0, [r1] + ; CHECK-NEXT: pop {r0, r1} + ; CHECK-NEXT: b [[CONTINUE:.*]] + ; CHECK-NEXT: .p2align 2 + ; CHECK-NEXT: [[LABEL]]: + ; CHECK-NEXT: .long _Z3foov$RAW-([[LOAD_LABEL]]+4) + ; CHECK-NEXT: [[CONTINUE]]: + + ret i32 0 + ; CHECK: bx lr +} + +; CHECK-LABEL: .section __llvm_mipraw + +; CHECK-LABEL: _Z3foov$RAW: +; CHECK-NEXT: .byte 0xff diff --git a/llvm/test/CodeGen/Thumb2/mip-function-coverage.ll b/llvm/test/CodeGen/Thumb2/mip-function-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/mip-function-coverage.ll @@ -0,0 +1,25 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=thumb-linux -mattr=+thumb2 -mcpu=cortex-a8 | FileCheck %s + +; CHECK-LABEL: _Z3foov: +define i32 @_Z3foov() #0 { + ; CHECK: push {r0, r1} + ; CHECK-NEXT: ldr r1, [[LABEL:.*]] + ; CHECK-NEXT: [[LOAD_LABEL:.*]]: + ; CHECK-NEXT: add r1, pc + ; CHECK-NEXT: movs r0, #0 + ; CHECK-NEXT: strb r0, [r1] + ; CHECK-NEXT: pop {r0, r1} + ; CHECK-NEXT: b [[CONTINUE:.*]] + ; CHECK-NEXT: .p2align 2 + ; CHECK-NEXT: [[LABEL]]: + ; CHECK-NEXT: .long _Z3foov$RAW-([[LOAD_LABEL]]+4) + ; CHECK-NEXT: [[CONTINUE]]: + + ret i32 0 + ; CHECK: bx lr +} + +; CHECK-LABEL: .section __llvm_mipraw + +; CHECK-LABEL: _Z3foov$RAW: +; CHECK-NEXT: .byte 0xff diff --git a/llvm/test/CodeGen/X86/mip-basic-block-coverage.ll b/llvm/test/CodeGen/X86/mip-basic-block-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/mip-basic-block-coverage.ll @@ -0,0 +1,56 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=x86_64-LINUX | FileCheck %s --check-prefixes CHECK,CHECK-ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefixes CHECK,CHECK-MACHO + +; CHECK-ELF-LABEL: _Z3fooi: +; CHECK-MACHO-LABEL: __Z3fooi: +define i32 @_Z3fooi(i32 %a) { +entry: + %a.addr = alloca i32, align 4 + %sum = alloca i32, align 4 + %i = alloca i32, align 4 + store i32 %a, i32* %a.addr, align 4 + store i32 0, i32* %sum, align 4 + store i32 0, i32* %i, align 4 + br label %for.cond + +for.cond: ; preds = %for.inc, %entry + ; CHECK-ELF: movb $0, _Z3fooi$RAW+1(%rip) + ; CHECK-MACHO: movb $0, __Z3fooi$RAW+1(%rip) + %0 = load i32, i32* %i, align 4 + %1 = load i32, i32* %a.addr, align 4 + %cmp = icmp slt i32 %0, %1 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + ; CHECK-ELF: movb $0, _Z3fooi$RAW+2(%rip) + ; CHECK-MACHO: movb $0, __Z3fooi$RAW+2(%rip) + %2 = load i32, i32* %i, align 4 + %3 = load i32, i32* %sum, align 4 + %add = add nsw i32 %3, %2 + store i32 %add, i32* %sum, align 4 + br label %for.inc + +for.inc: ; preds = %for.body + ; CHECK-ELF: movb $0, _Z3fooi$RAW+3(%rip) + ; CHECK-MACHO: movb $0, __Z3fooi$RAW+3(%rip) + %4 = load i32, i32* %i, align 4 + %inc = add nsw i32 %4, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + %5 = load i32, i32* %sum, align 4 + ret i32 %5 + ; CHECK: retq +} + +; CHECK-ELF-LABEL: .section __llvm_mipraw +; CHECK-MACHO-LABEL: .section __DATA,__llvm_mipraw + +; CHECK-ELF-LABEL: _Z3fooi$RAW: +; CHECK-ELF-NEXT: .byte 0xff +; CHECK-ELF-NEXT: .zero 3,255 + +; CHECK-MACHO-LABEL: __Z3fooi$RAW: +; CHECK-MACHO-NEXT: .byte 0xff +; CHECK-MACHO-NEXT: .space 3,255 diff --git a/llvm/test/CodeGen/X86/mip-comdat.ll b/llvm/test/CodeGen/X86/mip-comdat.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/mip-comdat.ll @@ -0,0 +1,12 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-linux | FileCheck %s + +; CHECK-DAG: .section .text.foo,"axG",@progbits,foo,comdat +$foo = comdat any +define i32 @foo(i32) comdat { + ret i32 101 +} + + +; CHECK-DAG: .section __llvm_mipraw,"aGw",@progbits,foo,comdat + +; CHECK-DAG: .section __llvm_mipmap,"aGw",@progbits,foo,comdat diff --git a/llvm/test/CodeGen/X86/mip-function-coverage.ll b/llvm/test/CodeGen/X86/mip-function-coverage.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/mip-function-coverage.ll @@ -0,0 +1,18 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefixes CHECK,CHECK-MACHO + +; CHECK-ELF-LABEL: _Z3foov: +; CHECK-MACHO-LABEL: __Z3foov: +define i32 @_Z3foov() #0 { + ; CHECK-ELF: movb $0, _Z3foov$RAW(%rip) + ; CHECK-MACHO: movb $0, __Z3foov$RAW(%rip) + ret i32 0 + ; CHECK: retq +} + +; CHECK-ELF-LABEL: .section __llvm_mipraw +; CHECK-MACHO-LABEL: .section __DATA,__llvm_mipraw + +; CHECK-ELF-LABEL: _Z3foov$RAW: +; CHECK-MACHO-LABEL: __Z3foov$RAW: +; CHECK-NEXT: .byte 0xff diff --git a/llvm/test/CodeGen/X86/mip-header.ll b/llvm/test/CodeGen/X86/mip-header.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/mip-header.ll @@ -0,0 +1,59 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-linux | FileCheck %s --check-prefix ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefix MACHO + +define i32 @_Z3fooii(i32 %a, i32 %b) #0 { + ret i32 0 +} + +;================================= .mipraw Header =====================================; +; ELF-LABEL: .section __llvm_mipraw,"aGw",@progbits,Header,comdat +; ELF: .p2align 3 +; ELF: [[RAW_REF:.*]]: +; ELF-NEXT: .long 0x50494dfb # Magic +; ELF-NEXT: .short 8 # Version +; ELF-NEXT: .short 0x1 # File Type +; ELF-NEXT: .long 0x1 # Profile Type +; ELF-NEXT: .long [[MODULE_HASH:.*]] # Module Hash +; ELF-NEXT: .zero 4 +; ELF-NEXT: .long 0x18 # Offset To Data + +;================================= .mipmap Header =====================================; +; ELF-LABEL: .section __llvm_mipmap,"aGw",@progbits,Header,comdat +; ELF: .p2align 3 +; ELF: [[MAP_REF:.*]]: +; ELF-NEXT: .long 0x50494dfb # Magic +; ELF-NEXT: .short 8 # Version +; ELF-NEXT: .short 0x2 # File Type +; ELF-NEXT: .long 0x1 # Profile Type +; ELF-NEXT: .long [[MODULE_HASH]] # Module Hash +; ELF-NEXT: .zero 4 +; ELF-NEXT: .long 0x18 # Offset To Data + +;================================= .mipraw Header =====================================; +; MACHO: .section __DATA,__llvm_mipraw,regular,no_dead_strip +; MACHO: .globl __header$__llvm_mipraw +; MACHO: .weak_definition __header$__llvm_mipraw +; MACHO-LABEL: __header$__llvm_mipraw: +; MACHO: .p2align 3 +; MACHO: .long 0x50494dfb ## Magic +; MACHO-NEXT: .short 8 ## Version +; MACHO-NEXT: .short 0x1 ## File Type +; MACHO-NEXT: .long 0x1 ## Profile Type +; MACHO-NEXT: .long [[MODULE_HASH:.*]] ## Module Hash +; MACHO-NEXT: .space 4 +; MACHO-NEXT: .long 0x18 ## Offset To Data + + +;================================= .mipmap Header =====================================; +; MACHO: .section __DATA,__llvm_mipmap,regular,no_dead_strip +; MACHO: .globl __header$__llvm_mipmap +; MACHO: .weak_definition __header$__llvm_mipmap +; MACHO-LABEL: __header$__llvm_mipmap: +; MACHO: .p2align 3 +; MACHO: .long 0x50494dfb ## Magic +; MACHO-NEXT: .short 8 ## Version +; MACHO-NEXT: .short 0x2 ## File Type +; MACHO-NEXT: .long 0x1 ## Profile Type +; MACHO-NEXT: .long [[MODULE_HASH]] ## Module Hash +; MACHO-NEXT: .space 4 +; MACHO-NEXT: .long 0x18 ## Offset To Data diff --git a/llvm/test/CodeGen/X86/mip-map.ll b/llvm/test/CodeGen/X86/mip-map.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/mip-map.ll @@ -0,0 +1,92 @@ +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=x86_64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF +; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -enable-machine-block-coverage -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefixes CHECK,CHECK-MACHO + +@global = local_unnamed_addr global i32 4, align 4 + +; CHECK-MACHO-LABEL: __Z3fooi: +; CHECK-ELF-LABEL: _Z3fooi: +define i32 @_Z3fooi(i32) { + ret i32 101 +} +; CHECK: [[FOO_END:.?Lmip_func_end[0-9]+]]: + +; CHECK-MACHO-LABEL: __Z3gooi: +; CHECK-ELF-LABEL: _Z3gooi: +define weak i32 @_Z3gooi(i32 %a) local_unnamed_addr #0 { +entry: + %cmp = icmp sgt i32 %a, 1 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry +; CHECK: [[GOO_BLOCK0:.?LBB.*]]: + %0 = load i32, i32* @global, align 4 + %call = tail call i32 @_Z3fooi(i32 1) #2 + %add.neg = sub i32 %a, %0 + %sub = sub i32 %add.neg, %call + br label %if.end + +if.end: ; preds = %if.then, %entry +; CHECK: [[GOO_BLOCK1:.?LBB.*]]: + %c.0 = phi i32 [ %sub, %if.then ], [ %a, %entry ] + %mul = mul nsw i32 %c.0, %a + ret i32 %mul +} +; CHECK: [[GOO_END:.?Lmip_func_end[0-9]+]]: + +attributes #0 = { minsize norecurse optsize ssp uwtable "frame-pointer"="non-leaf" } + +; CHECK-ELF-LABEL: .section __llvm_mipmap,"aw",@progbits + +; CHECK-ELF: .p2align 3 +; CHECK-ELF-LABEL: _Z3fooi$MAP: +; CHECK-ELF-NEXT: [[FOO_REF:.*]]: +; CHECK-ELF-NEXT: .long __start___llvm_mipraw-[[FOO_REF]] # Raw Section Start PC Offset +; CHECK-ELF-NEXT: .long _Z3fooi$RAW-[[FOO_REF]] # Raw Profile Symbol PC Offset +; CHECK-ELF-NEXT: .long _Z3fooi-[[FOO_REF]] # Function PC Offset +; CHECK-ELF-NEXT: .long [[FOO_END]]-_Z3fooi # Function Size +; CHECK-ELF-NEXT: .long 0x0 # CFG Signature +; CHECK-ELF-NEXT: .long 0 # Non-entry Block Count +; CHECK-ELF-NEXT: .long 7 # Function Name Length +; CHECK-ELF-NEXT: .ascii "_Z3fooi" + +; CHECK-ELF: .p2align 3 +; CHECK-ELF-LABEL: _Z3gooi$MAP: +; CHECK-ELF-NEXT: [[GOO_REF:.*]]: +; CHECK-ELF-NEXT: .long __start___llvm_mipraw-[[GOO_REF]] # Raw Section Start PC Offset +; CHECK-ELF-NEXT: .long _Z3gooi$RAW-[[GOO_REF]] # Raw Profile Symbol PC Offset +; CHECK-ELF-NEXT: .long _Z3gooi-[[GOO_REF]] # Function PC Offset +; CHECK-ELF-NEXT: .long [[GOO_END]]-_Z3gooi # Function Size +; CHECK-ELF-NEXT: .long 0x70c9fa27 # CFG Signature +; CHECK-ELF-NEXT: .long 2 # Non-entry Block Count +; CHECK-ELF-NEXT: .long [[GOO_BLOCK0]]-_Z3gooi # Block 0 Offset +; CHECK-ELF-NEXT: .long [[GOO_BLOCK1]]-_Z3gooi # Block 1 Offset +; CHECK-ELF-NEXT: .long 7 # Function Name Length +; CHECK-ELF-NEXT: .ascii "_Z3gooi" + +; CHECK-MACHO-LABEL: .section __DATA,__llvm_mipmap + +; CHECK-MACHO: .p2align 3 +; CHECK-MACHO-LABEL: __Z3fooi$MAP: +; CHECK-MACHO-NEXT: [[FOO_REF:.*]]: +; CHECK-MACHO-NEXT: .long __header$__llvm_mipraw-[[FOO_REF]] ## Raw Section Start PC Offset +; CHECK-MACHO-NEXT: .long __Z3fooi$RAW-[[FOO_REF]] ## Raw Profile Symbol PC Offset +; CHECK-MACHO-NEXT: .long __Z3fooi-[[FOO_REF]] ## Function PC Offset +; CHECK-MACHO-NEXT: .long [[FOO_END]]-__Z3fooi ## Function Size +; CHECK-MACHO-NEXT: .long 0x0 ## CFG Signature +; CHECK-MACHO-NEXT: .long 0 ## Non-entry Block Count +; CHECK-MACHO-NEXT: .long 8 ## Function Name Length +; CHECK-MACHO-NEXT: .ascii "__Z3fooi" + +; CHECK-MACHO: .p2align 3 +; CHECK-MACHO-LABEL: __Z3gooi$MAP: +; CHECK-MACHO-NEXT: [[GOO_REF:.*]]: +; CHECK-MACHO-NEXT: .long __header$__llvm_mipraw-[[GOO_REF]] ## Raw Section Start PC Offset +; CHECK-MACHO-NEXT: .long __Z3gooi$RAW-[[GOO_REF]] ## Raw Profile Symbol PC Offset +; CHECK-MACHO-NEXT: .long __Z3gooi-[[GOO_REF]] ## Function PC Offset +; CHECK-MACHO-NEXT: .long [[GOO_END]]-__Z3gooi ## Function Size +; CHECK-MACHO-NEXT: .long 0x70c9fa27 ## CFG Signature +; CHECK-MACHO-NEXT: .long 2 ## Non-entry Block Count +; CHECK-MACHO-NEXT: .long [[GOO_BLOCK0]]-__Z3gooi ## Block 0 Offset +; CHECK-MACHO-NEXT: .long [[GOO_BLOCK1]]-__Z3gooi ## Block 1 Offset +; CHECK-MACHO-NEXT: .long 8 ## Function Name Length +; CHECK-MACHO-NEXT: .ascii "__Z3gooi"