Index: include/llvm/CodeGen/AsmPrinter.h =================================================================== --- include/llvm/CodeGen/AsmPrinter.h +++ include/llvm/CodeGen/AsmPrinter.h @@ -242,7 +242,7 @@ }; // All the sleds to be emitted. - std::vector Sleds; + SmallVector Sleds; // Helper function to record a given XRay sled. void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind); Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -2761,37 +2761,63 @@ auto PrevSection = OutStreamer->getCurrentSectionOnly(); auto Fn = MF->getFunction(); - MCSection *Section = nullptr; + MCSection *InstMap = nullptr; + MCSection *FnSledIndex = nullptr; if (MF->getSubtarget().getTargetTriple().isOSBinFormatELF()) { if (Fn->hasComdat()) { - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + InstMap = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Fn->getComdat()->getName()); + FnSledIndex = OutContext.getELFSection("xray_fn_idx", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + Fn->getComdat()->getName()); } else { - Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + InstMap = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, ELF::SHF_ALLOC); + FnSledIndex = OutContext.getELFSection("xray_fn_idx", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC); } } else if (MF->getSubtarget().getTargetTriple().isOSBinFormatMachO()) { - Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, + InstMap = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, SectionKind::getReadOnlyWithRel()); + FnSledIndex = OutContext.getMachOSection("__DATA", "xray_fn_idx", 0, + SectionKind::getReadOnlyWithRel()); } else { llvm_unreachable("Unsupported target"); } // Before we switch over, we force a reference to a label inside the - // xray_instr_map section. Since this function is always called just - // before the function's end, we assume that this is happening after - // the last return instruction. - + // xray_instr_map and xray_fn_idx sections. Since this function is always + // called just before the function's end, we assume that this is happening + // after the last return instruction. We also use the synthetic label in the + // xray_inster_map as a delimeter for the range of sleds for this function in + // the index. auto WordSizeBytes = MAI->getCodePointerSize(); - MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true); + MCSymbol *SledsStart = OutContext.createTempSymbol("xray_synthetic_", true); + MCSymbol *IdxRef = OutContext.createTempSymbol("xray_fn_idx_synth_", true); OutStreamer->EmitCodeAlignment(16); - OutStreamer->EmitSymbolValue(Tmp, WordSizeBytes, false); - OutStreamer->SwitchSection(Section); - OutStreamer->EmitLabel(Tmp); + OutStreamer->EmitSymbolValue(SledsStart, WordSizeBytes, false); + OutStreamer->EmitSymbolValue(IdxRef, WordSizeBytes, false); + + // Now we switch to the instrumentation map section. Because this is done + // per-function, we are able to create an index entry that will represent the + // range of sleds associated with a function. + OutStreamer->SwitchSection(InstMap); + OutStreamer->EmitLabel(SledsStart); for (const auto &Sled : Sleds) Sled.emit(WordSizeBytes, OutStreamer.get(), CurrentFnSym); - + MCSymbol *SledsEnd = OutContext.createTempSymbol("xray_synthetic_end", true); + OutStreamer->EmitLabel(SledsEnd); + + // We then emit a single entry in the index per function. We use the symbols + // that bound the instrumentation map as the range for a specific function. + // Each entry here will be 16-byte aligned, as we're writing down two + // pointers. + OutStreamer->SwitchSection(FnSledIndex); + OutStreamer->EmitCodeAlignment(16); + OutStreamer->EmitLabel(IdxRef); + OutStreamer->EmitSymbolValue(SledsStart, WordSizeBytes); + OutStreamer->EmitSymbolValue(SledsEnd, WordSizeBytes); OutStreamer->SwitchSection(PrevSection); Sleds.clear(); } Index: test/CodeGen/AArch64/xray-attribute-instrumentation.ll =================================================================== --- test/CodeGen/AArch64/xray-attribute-instrumentation.ll +++ test/CodeGen/AArch64/xray-attribute-instrumentation.ll @@ -26,6 +26,7 @@ } ; CHECK: .p2align 4 ; CHECK-NEXT: .xword .Lxray_synthetic_0 +; CHECK-NEXT: .xword .Lxray_fn_idx_synth_0 ; CHECK-NEXT: .section xray_instr_map,{{.*}} ; CHECK-LABEL: Lxray_synthetic_0: ; CHECK: .xword .Lxray_sled_0 Index: test/CodeGen/AArch64/xray-tail-call-sled.ll =================================================================== --- test/CodeGen/AArch64/xray-tail-call-sled.ll +++ test/CodeGen/AArch64/xray-tail-call-sled.ll @@ -29,10 +29,16 @@ } ; CHECK: .p2align 4 ; CHECK-NEXT: .xword .Lxray_synthetic_0 +; CHECK-NEXT: .xword .Lxray_fn_idx_synth_0 ; CHECK-NEXT: .section xray_instr_map,{{.*}} ; CHECK-LABEL: Lxray_synthetic_0: ; CHECK: .xword .Lxray_sled_0 ; CHECK: .xword .Lxray_sled_1 +; CHECK-LABEL: Lxray_synthetic_end0: +; CHECK: .section xray_fn_idx,{{.*}} +; CHECK-LABEL: Lxray_fn_idx_synth_0: +; CHECK: .xword .Lxray_synthetic_0 +; CHECK-NEXT: .xword .Lxray_synthetic_end0 define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-always" { ; CHECK: .p2align 2 @@ -63,7 +69,13 @@ } ; CHECK: .p2align 4 ; CHECK-NEXT: .xword .Lxray_synthetic_1 +; CHECK-NEXT: .xword .Lxray_fn_idx_synth_1 ; CHECK-NEXT: .section xray_instr_map,{{.*}} ; CHECK-LABEL: Lxray_synthetic_1: ; CHECK: .xword .Lxray_sled_2 ; CHECK: .xword .Lxray_sled_3 +; CHECK-LABEL: Lxray_synthetic_end1: +; CHECK: .section xray_fn_idx,{{.*}} +; CHECK-LABEL: Lxray_fn_idx_synth_1: +; CHECK: .xword .Lxray_synthetic_1 +; CHECK-NEXT: .xword .Lxray_synthetic_end1 Index: test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll =================================================================== --- test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll +++ test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll @@ -25,7 +25,13 @@ } ; CHECK: .p2align 4 ; CHECK-NEXT: .long {{.*}}Lxray_synthetic_0 +; CHECK-NEXT: .long {{.*}}Lxray_fn_idx_synth_0 ; CHECK-NEXT: .section {{.*}}xray_instr_map{{.*}} ; CHECK-LABEL: Lxray_synthetic_0: ; CHECK: .long {{.*}}Lxray_sled_0 ; CHECK: .long {{.*}}Lxray_sled_1 +; CHECK-LABEL: Lxray_synthetic_end0: +; CHECK: .section {{.*}}xray_fn_idx{{.*}} +; CHECK-LABEL: Lxray_fn_idx_synth_0: +; CHECK: .long {{.*}}Lxray_synthetic_0 +; CHECK-NEXT: .long {{.*}}Lxray_synthetic_end0 Index: test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll =================================================================== --- test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll +++ test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll @@ -25,7 +25,14 @@ } ; CHECK: .p2align 4 ; CHECK-NEXT: .long {{.*}}Lxray_synthetic_0 +; CHECK-NEXT: .long {{.*}}Lxray_fn_idx_synth_0 ; CHECK-NEXT: .section {{.*}}xray_instr_map{{.*}} ; CHECK-LABEL: Lxray_synthetic_0: ; CHECK: .long {{.*}}Lxray_sled_0 ; CHECK: .long {{.*}}Lxray_sled_1 +; CHECK-LABEL: Lxray_synthetic_end0: +; CHECK: .section {{.*}}xray_fn_idx{{.*}} +; CHECK-LABEL: Lxray_fn_idx_synth_0: +; CHECK: .long {{.*}}xray_synthetic_0 +; CHECK-NEXT: .long {{.*}}xray_synthetic_end0 + Index: test/CodeGen/X86/xray-attribute-instrumentation.ll =================================================================== --- test/CodeGen/X86/xray-attribute-instrumentation.ll +++ test/CodeGen/X86/xray-attribute-instrumentation.ll @@ -15,10 +15,17 @@ } ; CHECK: .p2align 4, 0x90 ; CHECK-NEXT: .quad {{.*}}xray_synthetic_0 +; CHECK-NEXT: .quad {{.*}}xray_fn_idx_synth_0 ; CHECK-NEXT: .section {{.*}}xray_instr_map ; CHECK-LABEL: Lxray_synthetic_0: ; CHECK: .quad {{.*}}xray_sled_0 ; CHECK: .quad {{.*}}xray_sled_1 +; CHECK-LABEL: Lxray_synthetic_end0: +; CHECK: .section {{.*}}xray_fn_idx +; CHECK-LABEL: Lxray_fn_idx_synth_0: +; CHECK: .quad {{.*}}xray_synthetic_0 +; CHECK-NEXT: .quad {{.*}}xray_synthetic_end0 + ; We test multiple returns in a single function to make sure we're getting all ; of them with XRay instrumentation. @@ -46,8 +53,14 @@ } ; CHECK: .p2align 4, 0x90 ; CHECK-NEXT: .quad {{.*}}xray_synthetic_1 +; CHECK-NEXT: .quad {{.*}}xray_fn_idx_synth_1 ; CHECK-NEXT: .section {{.*}}xray_instr_map ; CHECK-LABEL: Lxray_synthetic_1: ; CHECK: .quad {{.*}}xray_sled_2 ; CHECK: .quad {{.*}}xray_sled_3 ; CHECK: .quad {{.*}}xray_sled_4 +; CHECK-LABEL: Lxray_synthetic_end1: +; CHECK: .section {{.*}}xray_fn_idx +; CHECK-LABEL: Lxray_fn_idx_synth_1: +; CHECK: .quad {{.*}}xray_synthetic_1 +; CHECK-NEXT: .quad {{.*}}xray_synthetic_end1 Index: test/CodeGen/X86/xray-tail-call-sled.ll =================================================================== --- test/CodeGen/X86/xray-tail-call-sled.ll +++ test/CodeGen/X86/xray-tail-call-sled.ll @@ -14,11 +14,17 @@ ; CHECK-NEXT: nopw %cs:512(%rax,%rax) } ; CHECK: .p2align 4, 0x90 -; CHECK-NEXT: .quad {{.*}}xray_synthetic_0 +; CHECK-NEXT: .quad {{.*}}xray_synthetic_0{{.*}} +; CHECK-NEXT: .quad {{.*}}xray_fn_idx_synth_0{{.*}} ; CHECK-NEXT: .section {{.*}}xray_instr_map ; CHECK-LABEL: Lxray_synthetic_0: ; CHECK: .quad {{.*}}xray_sled_0 ; CHECK: .quad {{.*}}xray_sled_1 +; CHECK-LABEL: Lxray_synthetic_end0: +; CHECK-NEXT: .section {{.*}}xray_fn_idx +; CHECK-LABEL: Lxray_fn_idx_synth_0: +; CHECK: .quad {{.*}}xray_synthetic_0 +; CHECK-NEXT: .quad {{.*}}xray_synthetic_end0 define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-always" { ; CHECK: .p2align 1, 0x90 @@ -36,7 +42,13 @@ ret i32 %retval } ; CHECK: .p2align 4, 0x90 -; CHECK-NEXT: .quad {{.*}}xray_synthetic_1 +; CHECK-NEXT: .quad {{.*}}xray_synthetic_1{{.*}} +; CHECK-NEXT: .quad {{.*}}xray_fn_idx_synth_1{{.*}} ; CHECK-LABEL: Lxray_synthetic_1: ; CHECK: .quad {{.*}}xray_sled_2 ; CHECK: .quad {{.*}}xray_sled_3 +; CHECK-LABEL: Lxray_synthetic_end1: +; CHECK: .section {{.*}}xray_fn_idx +; CHECK-LABEL: Lxray_fn_idx_synth_1: +; CHECK: .quad {{.*}}xray_synthetic_1 +; CHECK: .quad {{.*}}xray_synthetic_end1