Add call count instrumentation for MIP

[ellis]

Summary

Add function timestamp and function call count instrumentation to MIP.
Use the -fmachine-profile-call-counts clang frontend flag to enable.

[ellis]

Update API for getRawProfileSymbol()

[ellis]

Fix name mangling for Mach-O and fix an x86_64 assembly bug for Mach-O.

[ellis]

Fix Mach-O compiler-rt lib

[ellis]

Update tests.

compiler-rt/lib/mip/CMakeLists.txt

 add_compiler_rt_component(mip)
 
 set(MIP_SOURCES
   MIPHelper.c
   MIPRuntime.cpp
 )
 
 set(MIP_HEADERS
   MIPHelper.h
 )
 
+set(aarch64_SOURCES
+  MIPHelperAArch64.S
+)
+
+set(arm64_SOURCES
+  ${aarch64_SOURCES}
+)
+
+set(x86_64_SOURCES
+  MIPHelperX86_64.S
+)
+
 include_directories(../../include)
 
+set(ASM_SOURCES "")
+foreach(arch ${MIP_SUPPORTED_ARCH})
+  list(APPEND ASM_SOURCES ${${arch}_SOURCES})
+  if(APPLE)
+    set_source_files_properties(
+      ${${arch}_SOURCES}
+      PROPERTIES COMPILE_FLAGS
+      "-arch ${arch}"
+    )
+  endif()
+endforeach()
+
 if(APPLE)
   add_compiler_rt_runtime(clang_rt.mip
     STATIC
     OS ${MIP_SUPPORTED_OS}
     ARCHS ${MIP_SUPPORTED_ARCH}
     CFLAGS ${EXTRA_FLAGS}
-    SOURCES ${MIP_SOURCES}
+    SOURCES ${MIP_SOURCES} ${ASM_SOURCES}
     ADDITIONAL_HEADERS ${MIP_HEADERS}
     PARENT_TARGET mip)
 else()
   add_compiler_rt_runtime(clang_rt.mip
     STATIC
     ARCHS ${MIP_SUPPORTED_ARCH}
     CFLAGS ${EXTRA_FLAGS}
-    SOURCES ${MIP_SOURCES}
+    SOURCES ${MIP_SOURCES} ${ASM_SOURCES}
     ADDITIONAL_HEADERS ${MIP_HEADERS}
     PARENT_TARGET mip)
 endif()

compiler-rt/lib/mip/MIPHelper.h

 void __llvm_dump_mip_profile(void);
 int __llvm_dump_mip_profile_with_filename(const char *Filename);
 
 void *__llvm_mip_profile_begin(void);
 void *__llvm_mip_profile_end(void);
 
 void __llvm_mip_runtime_initialize(void);
 
+typedef struct {
+  uint32_t CallCount;
+  uint32_t Timestamp;
+  // uint8_t BlockCoverage[BlockCount];
+} ProfileData_t;
+
 #endif // MIP_MIPHELPER_H

compiler-rt/lib/mip/MIPHelper.c

 /*===- MIPHelper.c - Machine IR Profile Runtime Helper --------------------===*\
 |*
 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 |* See https://llvm.org/LICENSE.txt for license information.
 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 |*
 \*===----------------------------------------------------------------------===*/
 
 #include "MIPHelper.h"
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+// The global timestamp value. Effectively records the number of unique
+// functions that have been called. A value of zero means instrumentation is
+// disabled.
+uint32_t __llvm_mip_global_timestamp = 1;
+
 void __llvm_mip_runtime_initialize(void) { atexit(__llvm_dump_mip_profile); }
 
 void __llvm_dump_mip_profile(void) {
   const char *Filename = getenv("LLVM_MIP_PROFILE_FILENAME");
   if (!Filename || !Filename[0])
     Filename = "default.mipraw";
 
   char *FormatSpecifierPtr;
if (BytesWritten != DataSize) {
     fclose(filep);
     return -1;
   }
 
   fclose(filep);
   return 0;
 }
 
+// Implements "Call Count" function instrumentation and returns the address of
+// `ProfileData`.
+// NOTE: This code is not thread-safe, but that is ok because absolute precision
+//       of `CallCount` and `Timestamp` is not critical.!
+void *
+__llvm_mip_call_counts_instrumentation_helper(ProfileData_t *ProfileData) {
+  if (__llvm_mip_global_timestamp) {
+    if (ProfileData->CallCount == 0xFFFFFFFF) {
+      // This function is called for the first time.
+      ProfileData->CallCount = 1;
+      ProfileData->Timestamp = __llvm_mip_global_timestamp;
+      // Since the number of functions << 2^32, a saturating add is not
+      // necessary.
+      __llvm_mip_global_timestamp += 1;
+    } else {
+      // This is not the first time this function has been called.
+      uint32_t NewCallCount = ProfileData->CallCount + 1;
+      if (NewCallCount != 0xFFFFFFFF)
+        ProfileData->CallCount = NewCallCount;
+    }
+  }
+  return ProfileData;
+}
+
 #ifdef __linux__
 #define MIP_RAW_SECTION_BEGIN_SYMBOL MIP_CONCAT(__start_, MIP_RAW_SECTION)
 #define MIP_RAW_SECTION_END_SYMBOL MIP_CONCAT(__stop_, MIP_RAW_SECTION)
 extern char MIP_RAW_SECTION_BEGIN_SYMBOL;
 extern char MIP_RAW_SECTION_END_SYMBOL;
 #endif // __linux__
 
 #ifdef __APPLE__

compiler-rt/lib/mip/MIPHelperAArch64.S

+#if defined(__aarch64__) || defined(__arm64__)
+#include "../sanitizer_common/sanitizer_asm.h"
+
+// The address of the raw profile symbol is expected to be in the x16 register.
+#define PROFILE_REGISTER x16
+
+    .text
+    .p2align 2
+
+.macro SAVE_STACK_FRAME
+    str     x8, [sp, #-0xE0]!
+    stp     x7, x6, [sp, #0x10]
+    stp     x5, x4, [sp, #0x20]
+    stp     x3, x2, [sp, #0x30]
+    stp     x1, x0, [sp, #0x40]
+    stp     q7, q6, [sp, #0x50]
+    stp     q5, q4, [sp, #0x70]
+    stp     q3, q2, [sp, #0x90]
+    stp     q1, q0, [sp, #0xB0]
+    stp     x29, x30, [sp, #0xD0]
+    add     x29, sp, #0xD0
+    CFI_DEF_CFA(w29, 0x10)
+    CFI_OFFSET(w30, -0x08)
+    CFI_OFFSET(w29, -0x10)
+.endm
+
+.macro RESTORE_STACK_FRAME
+    ldp     x29, x30, [sp, #0xD0]
+    ldp     q1, q0, [sp, #0xB0]
+    ldp     q3, q2, [sp, #0x90]
+    ldp     q5, q4, [sp, #0x70]
+    ldp     q7, q6, [sp, #0x50]
+    ldp     x1, x0, [sp, #0x40]
+    ldp     x3, x2, [sp, #0x30]
+    ldp     x5, x4, [sp, #0x20]
+    ldp     x7, x6, [sp, #0x10]
+    ldr     x8, [sp], #0xE0
+.endm
+
+// Call the C instrumentation helper for function call counts. Expected to be
+// called directly from the instrumented function.
+    .global ASM_SYMBOL(__llvm_mip_call_counts_caller)
+    ASM_TYPE_FUNCTION(ASM_SYMBOL(__llvm_mip_call_counts_caller))
+ASM_SYMBOL(__llvm_mip_call_counts_caller):
+    CFI_STARTPROC
+    SAVE_STACK_FRAME
+    mov     x0, PROFILE_REGISTER
+    // void *Helper(uint64_t ProfileDataAddress)
+    bl      ASM_SYMBOL(__llvm_mip_call_counts_instrumentation_helper)
+
+    RESTORE_STACK_FRAME
+    // Restore x29, x30 and return to the caller as if we were never called.
+    mov     x16, x30
+    ldp     x29, x30, [sp], #0x10
+    ret     x16
+    CFI_ENDPROC
+    ASM_SIZE(__llvm_mip_call_counts_caller)
+
+#endif // defined(__aarch64__) || defined(__arm64__)

compiler-rt/lib/mip/MIPHelperX86_64.S

+#if defined(__x86_64__)
+#include "../sanitizer_common/sanitizer_asm.h"
+
+// The address of the raw profile symbol is expected to be in the rax register.
+#define PROFILE_REGISTER %rax
+
+    .text
+    .p2align 2
+
+.macro ALIGN_STACK_16B
+#if defined(__APPLE__)
+    subq    $$8, %rsp
+#else
+    subq    $8, %rsp
+#endif
+    CFI_ADJUST_CFA_OFFSET(8)
+.endm
+
+.macro RESTORE_STACK_ALIGNMENT
+#if defined(__APPLE__)
+    addq    $$8, %rsp
+#else
+    addq    $8, %rsp
+#endif
+    CFI_ADJUST_CFA_OFFSET(-8)
+.endm
+
+.macro SAVE_STACK_FRAME
+    pushfq
+    CFI_ADJUST_CFA_OFFSET(8)
+#if defined(__APPLE__)
+    subq    $$240, %rsp
+#else
+    subq    $240, %rsp
+#endif
+    CFI_ADJUST_CFA_OFFSET(240)
+    movq    %rbp, 232(%rsp)
+    movupd  %xmm0, 216(%rsp)
+    movupd  %xmm1, 200(%rsp)
+    movupd  %xmm2, 184(%rsp)
+    movupd  %xmm3, 168(%rsp)
+    movupd  %xmm4, 152(%rsp)
+    movupd  %xmm5, 136(%rsp)
+    movupd  %xmm6, 120(%rsp)
+    movupd  %xmm7, 104(%rsp)
+    movq    %rdi, 96(%rsp)
+    movq    %rax, 88(%rsp)
+    movq    %rdx, 80(%rsp)
+    movq    %rsi, 72(%rsp)
+    movq    %rcx, 64(%rsp)
+    movq    %r8, 56(%rsp)
+    movq    %r9, 48(%rsp)
+    movq    %r10, 40(%rsp)
+    movq    %r11, 32(%rsp)
+    movq    %r12, 24(%rsp)
+    movq    %r13, 16(%rsp)
+    movq    %r14, 8(%rsp)
+    movq    %r15, 0(%rsp)
+.endm
+
+.macro RESTORE_STACK_FRAME
+    movq    232(%rsp), %rbp
+    movupd  216(%rsp), %xmm0
+    movupd  200(%rsp), %xmm1
+    movupd  184(%rsp), %xmm2
+    movupd  168(%rsp), %xmm3
+    movupd  152(%rsp), %xmm4
+    movupd  136(%rsp), %xmm5
+    movupd  120(%rsp), %xmm6
+    movupd  104(%rsp), %xmm7
+    movq    96(%rsp), %rdi
+    movq    88(%rsp), %rax
+    movq    80(%rsp), %rdx
+    movq    72(%rsp), %rsi
+    movq    64(%rsp), %rcx
+    movq    56(%rsp), %r8
+    movq    48(%rsp), %r9
+    movq    40(%rsp), %r10
+    movq    32(%rsp), %r11
+    movq    24(%rsp), %r12
+    movq    16(%rsp), %r13
+    movq    8(%rsp), %r14
+    movq    0(%rsp), %r15
+#if defined(__APPLE__)
+    addq    $$240, %rsp
+#else
+    addq    $240, %rsp
+#endif
+    CFI_ADJUST_CFA_OFFSET(-240)
+    popfq
+    CFI_ADJUST_CFA_OFFSET(-8)
+.endm
+
+// Call the C instrumentation helper for function call counts. Expected to be
+// called directly from the instrumented function.
+    .global ASM_SYMBOL(__llvm_mip_call_counts_caller)
+    ASM_TYPE_FUNCTION(ASM_SYMBOL(__llvm_mip_call_counts_caller))
+ASM_SYMBOL(__llvm_mip_call_counts_caller):
+    CFI_STARTPROC
+    // NOTE: When __llvm_mip_call_counts_caller is the very first instruction
+    //       of the instrumented function, the stack is already aligned.
+    // ALIGN_STACK_16B
+    SAVE_STACK_FRAME
+    movq    PROFILE_REGISTER, %rdi
+    // void *Helper(uint64_t ProfileDataAddress)
+    callq   ASM_SYMBOL(__llvm_mip_call_counts_instrumentation_helper)
+
+    RESTORE_STACK_FRAME
+    retq
+    // RESTORE_STACK_ALIGNMENT
+    CFI_ENDPROC
+    ASM_SIZE(__llvm_mip_call_counts_caller)
+
+#endif // defined(__x86_64__)

llvm/include/llvm/Support/TargetOpcodes.def

 /// flow graph signature and the number of machine basic blocks.
 HANDLE_TARGET_OPCODE(MIP_FUNCTION_INSTRUMENTATION_MARKER)
 
 /// MIP_FUNCTION_COVERAGE_INSTRUMENTATION - This instruction implements function
 /// coverage instrumentation. It takes as input a free temporary register if
 /// one exists.
 HANDLE_TARGET_OPCODE(MIP_FUNCTION_COVERAGE_INSTRUMENTATION)
 
+/// MIP_INSTRUMENTATION - This instruction implements mip instrumentation. It
+/// takes as input a free temporary register if one exists and the symbol to
+/// the helper function that implements the instrumentation.
+HANDLE_TARGET_OPCODE(MIP_INSTRUMENTATION)
+
 /// MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION - This instruction implements basic
 /// block coverage instrumentation. It takes as input a free temporary register
 /// if one exists and the id of the instrumented block.
 HANDLE_TARGET_OPCODE(MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION)
 
 /// KILL - This instruction is a noop that is used only to adjust the
 /// liveness of registers. This can be useful when dealing with
 /// sub-registers.

llvm/include/llvm/Target/Target.td

def MIP_FUNCTION_INSTRUMENTATION_MARKER : StandardPseudoInstruction {
   let hasSideEffects = 0;
 }
 def MIP_FUNCTION_COVERAGE_INSTRUMENTATION : StandardPseudoInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins unknown:$reg);
   let AsmString = "";
   let hasSideEffects = 1;
 }
+def MIP_INSTRUMENTATION : StandardPseudoInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins unknown:$reg, unknown:$helper);
+  let AsmString = "";
+  let hasSideEffects = 1;
+}
 def MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION : StandardPseudoInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins unknown:$reg, i32imm:$n);
   let AsmString = "";
   let hasSideEffects = 1;
 }
 def PHI : StandardPseudoInstruction {
   let OutOperandList = (outs unknown:$dst);

llvm/lib/CodeGen/MIPReader.cpp

if (Header.Magic != MIP_MAGIC_VALUE) {
     return std::make_error_code(std::errc::invalid_argument);
   } else if ((Header.FileType & 0b1111) != FileType) {
     WithColor::error() << Filename << ": Invalid file type\n";
     WithColor::error() << "Expected: " << Twine::utohexstr(FileType) << "\n";
     WithColor::error() << "     Got: " << Twine::utohexstr(Header.FileType)
                        << "\n";
     return std::make_error_code(std::errc::invalid_argument);
   } else if (Header.ProfileType & ~(MIP_PROFILE_TYPE_FUNCTION_COVERAGE |
-                                    MIP_PROFILE_TYPE_BLOCK_COVERAGE)) {
+                                    MIP_PROFILE_TYPE_BLOCK_COVERAGE |
+                                    MIP_PROFILE_TYPE_FUNCTION_TIMESTAMP |
+                                    MIP_PROFILE_TYPE_FUNCTION_CALL_COUNT)) {
     WithColor::error() << Filename << ": Invalid profile type\n"
                        << "Got: 0x" << Twine::utohexstr(Header.ProfileType)
                        << " \n";
     return std::make_error_code(std::errc::invalid_argument);
   } else {
     return std::move(Buffer);
   }
 }
if (Profile.RawProfileDataAddress >= Buffer->getBufferSize()) {
       return std::make_error_code(std::errc::invalid_argument);
     }
     const char *Data = Buffer->getBufferStart() + Profile.RawProfileDataAddress;
     MFRawProfile RawProfile;
     RawProfile.RawProfileDataAddress = Profile.RawProfileDataAddress;
     if (RawMIP->Header.ProfileType & MIP_PROFILE_TYPE_FUNCTION_COVERAGE) {
       RawProfile.IsFunctionCovered =
           (endian::readNext<uint8_t, little, unaligned>(Data) == 0x00);
+    } else {
+      uint32_t FunctionCallCount =
+          endian::readNext<uint32_t, little, unaligned>(Data);
+      uint32_t FunctionTimestamp =
+          endian::readNext<uint32_t, little, unaligned>(Data);
+      if ((FunctionCallCount == 0xFFFFFFFF) !=
+          (FunctionTimestamp == 0xFFFFFFFF)) {
+        WithColor::error() << "Corrupt raw profile near offset 0x"
+                           << Twine::utohexstr(
+                                  (uint64_t)(Data - Buffer->getBufferStart()))
+                           << "\n";
+        return std::make_error_code(std::errc::invalid_argument);
+      }
+      if (FunctionCallCount != 0xFFFFFFFF) {
+        RawProfile.FunctionCallCount = FunctionCallCount;
+        RawProfile.FunctionTimestamp = FunctionTimestamp;
+      }
+      RawProfile.IsFunctionCovered = (RawProfile.FunctionCallCount > 0);
     }
 
     if (RawMIP->Header.ProfileType & MIP_PROFILE_TYPE_BLOCK_COVERAGE) {
       // NOTE: The entry basic block profile is not in the raw file format.
       RawProfile.BasicBlockCoverage.push_back(RawProfile.IsFunctionCovered);
       for (size_t i = 1; i < Profile.BasicBlockProfiles.size(); i++) {
         bool IsBlockCovered =
             (endian::readNext<uint8_t, little, unaligned>(Data) == 0x00);

llvm/lib/CodeGen/MIRInstrumentationPass.cpp

if (EnableMachineFunctionCoverage == EnableMachineCallGraph)
                     Twine(EnableMachineInstrumentation.ArgStr) + ".");
 
     if (EnableMachineFunctionCoverage && MachineProfileRuntimeBufferSize)
       Ctx.emitError("Cannot set -" +
                     Twine(MachineProfileRuntimeBufferSize.ArgStr) + " when -" +
                     Twine(EnableMachineFunctionCoverage.ArgStr) +
                     " is provided.");
 
-    if (EnableMachineCallGraph)
-      Ctx.emitError("-" + Twine(EnableMachineCallGraph.ArgStr) +
-                    " is not yet implemented.");
-
     if (MachineProfileRuntimeBufferSize)
       Ctx.emitError("-" + Twine(MachineProfileRuntimeBufferSize.ArgStr) +
                     " is not yet implemented.");
   }
   return false;
 }
 
 bool MIRInstrumentation::runOnMachineFunction(MachineFunction &MF) {
BuildMI(EntryBlock, MBBI, DL,
       .addImm(NonEntryBlockCount);
   ++NumInstrumented;
 
   if (EnableMachineFunctionCoverage) {
     BuildMI(EntryBlock, MBBI, DL,
             TII.get(TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION))
         .addReg(TII.getTemporaryMachineProfileRegister(EntryBlock));
   } else if (EnableMachineCallGraph) {
-    llvm_unreachable("Not yet implemented");
+    BuildMI(EntryBlock, MBBI, DL, TII.get(TargetOpcode::MIP_INSTRUMENTATION))
+        .addReg(TII.getTemporaryMachineProfileRegister(EntryBlock))
+        .addExternalSymbol("__llvm_mip_call_counts_caller");
   } else {
     llvm_unreachable(
         "Expected function coverage or call graph instrumentation.");
   }
 
   for (uint32_t BlockID = 0; BlockID < NonEntryBlockCount; BlockID++) {
     auto &MBB = *MBBs[/*EntryBlock=*/1 + BlockID];
     runOnMachineBasicBlock(MBB, BlockID);

llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
                        const MachineInstr &MI);
   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
                        const MachineInstr &MI);
   void LowerFAULTING_OP(const MachineInstr &MI);
 
   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
   void LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(const MachineInstr &MI);
   void LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(const MachineInstr &MI);
+  void LowerMIP_INSTRUMENTATION(const MachineInstr &MI);
   void LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(const MachineInstr &MI);
   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
 
   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
   void EmitHwasanMemaccessSymbols(Module &M);
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADRP)
                                    .addOperand(RawAddressPageMCO));
   // strb   wzr, [<ProfileRegister>, <RawProfileSymbolPageOffset>]
   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::STRBBui)
                                    .addReg(AArch64::WZR)
                                    .addReg(ProfileRegister)
                                    .addOperand(RawAddressPageOffsetMCO));
 }
 
+void AArch64AsmPrinter::LowerMIP_INSTRUMENTATION(const MachineInstr &MI) {
+  auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(*MI.getMF());
+
+  const auto ProfileRegister = AArch64::X16;
+  auto RawAddressPageMO =
+      MachineOperand::CreateMCSymbol(RawProfileSymbol, AArch64II::MO_PAGE);
+  auto RawAddressPageOffsetMO = MachineOperand::CreateMCSymbol(
+      RawProfileSymbol, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+  auto HelperSymbolMO = MI.getOperand(1);
+  MCOperand RawAddressPageMCO, RawAddressPageOffsetMCO, HelperSymbolMCO;
+  lowerOperand(RawAddressPageMO, RawAddressPageMCO);
+  lowerOperand(RawAddressPageOffsetMO, RawAddressPageOffsetMCO);
+  lowerOperand(HelperSymbolMO, HelperSymbolMCO);
+
+  OutStreamer->AddComment("MIP: Instrumentation");
+  // stp    x29, x30, [sp, #-16]!
+  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::STPXpre)
+                                   .addReg(AArch64::SP)
+                                   .addReg(AArch64::FP)
+                                   .addReg(AArch64::LR)
+                                   .addReg(AArch64::SP)
+                                   .addImm(-2));
+  // adrp   <ProfileRegister>, <RawProfileSymbolPage>
+  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADRP)
+                                   .addReg(ProfileRegister)
+                                   .addOperand(RawAddressPageMCO));
+  // add    <ProfileRegister>, <ProfileRegister>, <RawProfileSymbolPageOffset>
+  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
+                                   .addReg(ProfileRegister)
+                                   .addReg(ProfileRegister)
+                                   .addOperand(RawAddressPageOffsetMCO)
+                                   .addImm(0));
+  // bl     <HelperSymbol>
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(AArch64::BL).addOperand(HelperSymbolMCO));
+}
+
 void AArch64AsmPrinter::LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(
     const MachineInstr &MI) {
   MIPEmitter.runOnBasicBlockInstrumentationMarker(MI);
 
   auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(*MI.getMF());
   auto TempRegister = MI.getOperand(0).getReg();
   auto BlockID = MI.getOperand(1).getImm();
   auto Offset = MIPEmitter.getOffsetToRawBlockProfileSymbol(BlockID);
case TargetOpcode::STACKMAP:
     return LowerSTACKMAP(*OutStreamer, SM, *MI);
 
   case TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER:
     return LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(*MI);
 
   case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION:
     return LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(*MI);
 
+  case TargetOpcode::MIP_INSTRUMENTATION:
+    return LowerMIP_INSTRUMENTATION(*MI);
+
   case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION:
     return LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(*MI);
 
   case TargetOpcode::PATCHPOINT:
     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
 
   case TargetOpcode::STATEPOINT:
     return LowerSTATEPOINT(*OutStreamer, SM, *MI);

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   switch (Desc.getOpcode()) {
   default:
     // Anything not explicitly designated otherwise is a normal 4-byte insn.
     NumBytes = 4;
     break;
   case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION:
     NumBytes = 8;
     break;
+  case TargetOpcode::MIP_INSTRUMENTATION:
+    NumBytes = 16;
+    break;
   case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION:
     NumBytes = (MI.getOperand(0).getReg() == AArch64::NoRegister) ? 20 : 12;
     break;
   case TargetOpcode::STACKMAP:
     // The upper bound for a stackmap intrinsic is the full length of its shadow
     NumBytes = StackMapOpers(&MI).getNumPatchBytes();
     assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
     break;

llvm/lib/Target/X86/X86AsmPrinter.h

class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
   void LowerSTATEPOINT(const MachineInstr &MI, X86MCInstLower &MCIL);
   void LowerFAULTING_OP(const MachineInstr &MI, X86MCInstLower &MCIL);
   void LowerPATCHABLE_OP(const MachineInstr &MI, X86MCInstLower &MCIL);
 
   void LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI);
 
   void LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(const MachineInstr &MI);
   void LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(const MachineInstr &MI);
+  void LowerMIP_INSTRUMENTATION(const MachineInstr &MI, X86MCInstLower &MCIL);
   void LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(const MachineInstr &MI);
 
   // XRay-specific lowering for X86.
   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
                                      X86MCInstLower &MCIL);
   void LowerPATCHABLE_RET(const MachineInstr &MI, X86MCInstLower &MCIL);
   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, X86MCInstLower &MCIL);
   void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, X86MCInstLower &MCIL);

llvm/lib/Target/X86/X86MCInstLower.cpp

EmitAndCountInstruction(
           .addImm(1)        // ScaleAmt
           .addReg(0)        // IndexReg
           .addExpr(MCSymbolRefExpr::create(RawProfileSymbol, OutContext))
           .addReg(0) // Segment
           .addImm(0) // Immediate
   );
 }
 
+void X86AsmPrinter::LowerMIP_INSTRUMENTATION(const MachineInstr &MI,
+                                             X86MCInstLower &MCIL) {
+  auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(*MI.getMF());
+  Optional<MCOperand> HelperSymbolOp =
+      MCIL.LowerMachineOperand(&MI, MI.getOperand(1));
+
+  OutStreamer->AddComment("MIP Instrumentation");
+  // leaq   <RawProfileSymbol>(%rip), %rax
+  EmitAndCountInstruction(
+      MCInstBuilder(X86::LEA64r)
+          .addReg(X86::RAX)
+          .addReg(X86::RIP)
+          .addImm(1)
+          .addReg(X86::NoRegister)
+          .addExpr(MCSymbolRefExpr::create(RawProfileSymbol, OutContext))
+          .addReg(X86::NoRegister));
+
+  // callq  <HelperSymbol>
+  EmitAndCountInstruction(
+      MCInstBuilder(X86::CALL64pcrel32).addOperand(*HelperSymbolOp));
+}
+
 void X86AsmPrinter::LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(
     const MachineInstr &MI) {
   MIPEmitter.runOnBasicBlockInstrumentationMarker(MI);
 
   auto *RawProfileSymbol = MIPEmitter.getRawProfileSymbol(*MI.getMF());
 
   auto BlockID = MI.getOperand(1).getImm();
   auto Offset = MIPEmitter.getOffsetToRawBlockProfileSymbol(BlockID);
case TargetOpcode::STACKMAP:
     return LowerSTACKMAP(*MI);
 
   case TargetOpcode::MIP_FUNCTION_INSTRUMENTATION_MARKER:
     return LowerMIP_FUNCTION_INSTRUMENTATION_MARKER(*MI);
 
   case TargetOpcode::MIP_FUNCTION_COVERAGE_INSTRUMENTATION:
     return LowerMIP_FUNCTION_COVERAGE_INSTRUMENTATION(*MI);
 
+  case TargetOpcode::MIP_INSTRUMENTATION:
+    return LowerMIP_INSTRUMENTATION(*MI, MCInstLowering);
+
   case TargetOpcode::MIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION:
     return LowerMIP_BASIC_BLOCK_COVERAGE_INSTRUMENTATION(*MI);
 
   case TargetOpcode::PATCHPOINT:
     return LowerPATCHPOINT(*MI, MCInstLowering);
 
   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
     return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);

llvm/test/CodeGen/AArch64/mip-call-counts.ll

+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=arm64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF
+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=arm64-apple-ios | FileCheck %s --check-prefixes CHECK,CHECK-MACHO
+
+; CHECK-ELF-LABEL: _Z3foov:
+; CHECK-MACHO-LABEL: __Z3foov:
+define i32 @_Z3foov() #0 {
+  ; CHECK:      stp     x29, x30, [sp, #-16]!
+  ; CHECK-ELF-NEXT: adrp    [[r:x[0-9]+]], _Z3foov$RAW
+  ; CHECK-ELF-NEXT: add     [[r]], [[r]], :lo12:_Z3foov$RAW
+  ; CHECK-ELF-NEXT: bl      __llvm_mip_call_counts_caller
+
+  ; CHECK-MACHO-NEXT: adrp    [[r:x[0-9]+]], __Z3foov$RAW@PAGE
+  ; CHECK-MACHO-NEXT: add     [[r]], [[r]], __Z3foov$RAW@PAGEOFF
+  ; CHECK-MACHO-NEXT: bl      ___llvm_mip_call_counts_caller
+  ret i32 0
+  ; CHECK: ret
+}
+
+; CHECK-ELF-LABEL: .section        __llvm_mipraw,"aGw",@progbits,"_Z3foov$MIP"
+; CHECK-MACHO-LABEL: .section        __DATA,__llvm_mipraw
+
+; CHECK-ELF:       .p2align 2
+; CHECK-ELF-LABEL: _Z3foov$RAW:
+; CHECK-ELF-NEXT:  .word    0xffffffff
+; CHECK-ELF-NEXT:  .word    0xffffffff
+
+; CHECK-MACHO:       .p2align   2
+; CHECK-MACHO-LABEL: __Z3foov$RAW:
+; CHECK-MACHO-NEXT:  .long      0xffffffff
+; CHECK-MACHO-NEXT:  .long      0xffffffff

llvm/test/CodeGen/AArch64/mip-header.ll

-; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-linux | FileCheck %s --check-prefix ELF
-; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=arm64-apple-ios | FileCheck %s --check-prefix MACHO
+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=arm64-linux | FileCheck %s --check-prefix ELF
+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=arm64-apple-ios | FileCheck %s --check-prefix MACHO
 
 define i32 @_Z3fooii(i32 %a, i32 %b) #0 {
   ret i32 0
 }
 
 ;================================= .mipraw Header =====================================;
 ; ELF:       .section __llvm_mipraw,"aGw",@progbits,Header,comdat
 ; ELF:       .p2align 3
 ; ELF:       .word   0x50494dfb                      // Magic
 ; ELF-NEXT:  .hword  8                               // Version
 ; ELF-NEXT:  .hword  0x11                            // File Type
-; ELF-NEXT:  .word   0x1                             // Profile Type
+; ELF-NEXT:  .word   0xc                             // Profile Type
 ; ELF-NEXT:  .word   [[MODULE_HASH:.*]]              // Module Hash
 ; ELF-NEXT:  .zero   8
 ; ELF-NEXT:  .zero   4
 ; ELF-NEXT:  .word   0x20                            // Offset To Data
 
 ;================================= .mipmap Header =====================================;
 ; ELF:       .section __llvm_mipmap,"GwR",@progbits,Header,comdat
 ; ELF:       .p2align 3
 ; ELF:       .word   0x50494dfb                      // Magic
 ; ELF-NEXT:  .hword  8                               // Version
 ; ELF-NEXT:  .hword  0x14                            // File Type
-; ELF-NEXT:  .word   0x1                             // Profile Type
+; ELF-NEXT:  .word   0xc                             // Profile Type
 ; ELF-NEXT:  .word   [[MODULE_HASH]]                 // Module Hash
 ; ELF-NEXT:  .zero   8
 ; ELF-NEXT:  .zero   4
 ; ELF-NEXT:  .word   0x20                            // Offset To Data
 
 ;================================= .mipraw Header =====================================;
 ; MACHO:       .section __DATA,__llvm_mipraw,regular,no_dead_strip
 ; MACHO:       .weak_definition __header$__llvm_mipraw
 ; MACHO-LABEL: __header$__llvm_mipraw:
 ; MACHO:       .p2align 3
 ; MACHO:       .long     0x50494dfb              ; Magic
 ; MACHO-NEXT:  .short    8                       ; Version
 ; MACHO-NEXT:  .short    0x11                    ; File Type
-; MACHO-NEXT:  .long     0x1                     ; Profile Type
+; MACHO-NEXT:  .long     0xc                     ; Profile Type
 ; MACHO-NEXT:  .long     [[MODULE_HASH:.*]]      ; Module Hash
 ; MACHO-NEXT:  .space    8
 ; MACHO-NEXT:  .space    4
 ; MACHO-NEXT:  .long     0x20                    ; Offset To Data
 
 ;================================= .mipmap Header =====================================;
 ; MACHO:       .section __DATA,__llvm_mipmap,regular,no_dead_strip
 ; MACHO:       .weak_definition __header$__llvm_mipmap
 ; MACHO-LABEL: __header$__llvm_mipmap:
 ; MACHO:       .p2align 3
 ; MACHO:       .long     0x50494dfb              ; Magic
 ; MACHO-NEXT:  .short    8                       ; Version
 ; MACHO-NEXT:  .short    0x14                    ; File Type
-; MACHO-NEXT:  .long     0x1                     ; Profile Type
+; MACHO-NEXT:  .long     0xc                     ; Profile Type
 ; MACHO-NEXT:  .long     [[MODULE_HASH]]         ; Module Hash
 ; MACHO-NEXT:  .space    8
 ; MACHO-NEXT:  .space    4
 ; MACHO-NEXT:  .long     0x20                    ; Offset To Data

llvm/test/CodeGen/X86/mip-call-counts.ll

+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=x86_64-linux | FileCheck %s --check-prefixes CHECK,CHECK-ELF
+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefixes CHECK,CHECK-MACHO
+
+; CHECK-ELF-LABEL: _Z3foov:
+; CHECK-MACHO-LABEL: __Z3foov:
+define i32 @_Z3foov() #0 {
+  ; CHECK-ELF:      leaq    _Z3foov$RAW(%rip), %rax
+  ; CHECK-ELF-NEXT: callq   __llvm_mip_call_counts_caller
+  ; CHECK-MACHO:      leaq    __Z3foov$RAW(%rip), %rax
+  ; CHECK-MACHO-NEXT: callq   ___llvm_mip_call_counts_caller
+  ret i32 0
+  ; CHECK: retq
+}
+
+; CHECK-ELF-LABEL: .section        __llvm_mipraw,"aGw",@progbits,"_Z3foov$MIP"
+; CHECK-MACHO-LABEL: .section        __DATA,__llvm_mipraw
+
+; CHECK:  .p2align   2
+; CHECK-ELF-LABEL: _Z3foov$RAW:
+; CHECK-MACHO-LABEL: __Z3foov$RAW:
+
+; CHECK-NEXT:  .long      0xffffffff
+; CHECK-NEXT:  .long      0xffffffff

llvm/test/CodeGen/X86/mip-header.ll

-; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-linux | FileCheck %s --check-prefix ELF
-; RUN: llc < %s -enable-machine-instrumentation -enable-machine-function-coverage -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefix MACHO
+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=x86_64-linux | FileCheck %s --check-prefix ELF
+; RUN: llc < %s -enable-machine-instrumentation -enable-machine-call-graph -mtriple=x86_64-apple-macosx | FileCheck %s --check-prefix MACHO
 
 define i32 @_Z3fooii(i32 %a, i32 %b) #0 {
   ret i32 0
 }
 
 ;================================= .mipraw Header =====================================;
 ; ELF-LABEL:   .section  __llvm_mipraw,"aGw",@progbits,Header,comdat
 ; ELF:         .p2align  3
 ; ELF-NEXT:    .long    0x50494dfb                      # Magic
 ; ELF-NEXT:    .short   8                               # Version
 ; ELF-NEXT:    .short   0x11                            # File Type
-; ELF-NEXT:    .long    0x1                             # Profile Type
+; ELF-NEXT:    .long    0xc                             # Profile Type
 ; ELF-NEXT:    .long    [[MODULE_HASH:.*]]              # Module Hash
 ; ELF-NEXT:    .zero    8
 ; ELF-NEXT:    .zero    4
 ; ELF-NEXT:    .long    0x20                            # Offset To Data
 
 ;================================= .mipmap Header =====================================;
 ; ELF-LABEL:   .section  __llvm_mipmap,"GwR",@progbits,Header,comdat
 ; ELF:         .p2align  3
 ; ELF-NEXT:    .long    0x50494dfb                      # Magic
 ; ELF-NEXT:    .short   8                               # Version
 ; ELF-NEXT:    .short   0x14                            # File Type
-; ELF-NEXT:    .long    0x1                             # Profile Type
+; ELF-NEXT:    .long    0xc                             # Profile Type
 ; ELF-NEXT:    .long    [[MODULE_HASH]]                 # Module Hash
 ; ELF-NEXT:    .zero    8
 ; ELF-NEXT:    .zero    4
 ; ELF-NEXT:    .long    0x20                            # Offset To Data
 
 ;================================= .mipraw Header =====================================;
 ; MACHO:       .section __DATA,__llvm_mipraw,regular,no_dead_strip
 ; MACHO:       .globl   __header$__llvm_mipraw
 ; MACHO:       .weak_definition  __header$__llvm_mipraw
 ; MACHO-LABEL: __header$__llvm_mipraw:
 ; MACHO:       .p2align 3
 ; MACHO:       .long   0x50494dfb              ## Magic
 ; MACHO-NEXT:  .short  8                       ## Version
 ; MACHO-NEXT:  .short  0x11                    ## File Type
-; MACHO-NEXT:  .long   0x1                     ## Profile Type
+; MACHO-NEXT:  .long   0xc                     ## Profile Type
 ; MACHO-NEXT:  .long   [[MODULE_HASH:.*]]      ## Module Hash
 ; MACHO-NEXT:  .space  8
 ; MACHO-NEXT:  .space  4
 ; MACHO-NEXT:  .long   0x20                    ## Offset To Data
 
 
 ;================================= .mipmap Header =====================================;
 ; MACHO:       .section __DATA,__llvm_mipmap,regular,no_dead_strip
 ; MACHO:       .globl   __header$__llvm_mipmap
 ; MACHO:       .weak_definition  __header$__llvm_mipmap
 ; MACHO-LABEL: __header$__llvm_mipmap:
 ; MACHO:       .p2align 3
 ; MACHO:       .long   0x50494dfb              ## Magic
 ; MACHO-NEXT:  .short  8                       ## Version
 ; MACHO-NEXT:  .short  0x14                    ## File Type
-; MACHO-NEXT:  .long   0x1                     ## Profile Type
+; MACHO-NEXT:  .long   0xc                     ## Profile Type
 ; MACHO-NEXT:  .long   [[MODULE_HASH]]         ## Module Hash
 ; MACHO-NEXT:  .space  8
 ; MACHO-NEXT:  .space  4
 ; MACHO-NEXT:  .long   0x20                    ## Offset To Data

llvm/tools/llvm-mipdata/llvm-mipdata.cpp

for (auto &RawProfile : RawMIP->RawProfiles) {
           MIP->Profiles[ProfileMap[RawProfile.RawProfileDataAddress]];
 
       if (RawHeader.ProfileType & MIP_PROFILE_TYPE_FUNCTION_COVERAGE) {
         if (RawProfile.IsFunctionCovered) {
           Profile.RawProfileCount++;
           Profile.FunctionCallCount =
               std::max<uint64_t>(Profile.FunctionCallCount, 1);
         }
+      } else {
+        if (RawProfile.IsFunctionCovered) {
+          Profile.RawProfileCount++;
+          Profile.FunctionCallCount += RawProfile.FunctionCallCount;
+          Profile.FunctionOrderSum += RawProfile.FunctionTimestamp;
+        }
       }
 
       if (RawHeader.ProfileType & MIP_PROFILE_TYPE_BLOCK_COVERAGE) {
         assert(RawProfile.BasicBlockCoverage.size() ==
                Profile.BasicBlockProfiles.size());
         for (unsigned BlockID = 0;
              BlockID < RawProfile.BasicBlockCoverage.size(); BlockID++) {
           Profile.BasicBlockProfiles[BlockID].IsCovered =
if (SymReader) {
         SourceInfo = InliningInfo.get().getFrame(0);
     }
 
     OS << Profile.FunctionName << "\n";
     if (SourceInfo)
       OS << "  Source Info: " << SourceInfo.FileName << ":" << SourceInfo.Line
          << "\n";
     OS << "  Call Count: " << Profile.FunctionCallCount << "\n";
+    if (Profile.FunctionOrderSum)
+      OS << "  Order Sum: " << Profile.FunctionOrderSum << "\n";
     if (Profile.BasicBlockProfiles.size() > 1) {
       OS << "  Block Coverage:";
       for (unsigned I = 0; I < Profile.BasicBlockProfiles.size(); I++) {
         const auto &BlockProfile = Profile.BasicBlockProfiles[I];
         if (I % 8 == 0) {
           OS << "\n    ";
         }
         if (BlockProfile.IsCovered) {