Index: llvm/lib/CodeGen/XRayInstrumentation.cpp =================================================================== --- llvm/lib/CodeGen/XRayInstrumentation.cpp +++ llvm/lib/CodeGen/XRayInstrumentation.cpp @@ -34,6 +34,15 @@ namespace { +struct Option { + // Whether to emit PATCHABLE_TAIL_CALL. + bool HandleTailcall; + + // Whether to emit PATCHABLE_RET/PATCHABLE_FUNCTION_EXIT for all forms of + // return, e.g. conditional return. + bool HandleAllReturns; +}; + struct XRayInstrumentation : public MachineFunctionPass { static char ID; @@ -59,7 +68,7 @@ // This is the approach to go on CPUs which have a single RET instruction, // like x86/x86_64. void replaceRetWithPatchableRet(MachineFunction &MF, - const TargetInstrInfo *TII); + const TargetInstrInfo *TII, Option); // Prepend the original return instruction with the exit sled code ("patchable // function exit" pseudo-instruction), preserving the original return @@ -70,25 +79,27 @@ // have to call the trampoline and return from it to the original return // instruction of the function being instrumented. void prependRetWithPatchableExit(MachineFunction &MF, - const TargetInstrInfo *TII); + const TargetInstrInfo *TII, Option); }; } // end anonymous namespace -void XRayInstrumentation::replaceRetWithPatchableRet( - MachineFunction &MF, const TargetInstrInfo *TII) { +void XRayInstrumentation::replaceRetWithPatchableRet(MachineFunction &MF, + const TargetInstrInfo *TII, + Option op) { // We look for *all* terminators and returns, then replace those with // PATCHABLE_RET instructions. SmallVector Terminators; for (auto &MBB : MF) { for (auto &T : MBB.terminators()) { unsigned Opc = 0; - if (T.isReturn() && T.getOpcode() == TII->getReturnOpcode()) { + if (T.isReturn() && + (op.HandleAllReturns || T.getOpcode() == TII->getReturnOpcode())) { // Replace return instructions with: // PATCHABLE_RET , ... Opc = TargetOpcode::PATCHABLE_RET; } - if (TII->isTailCall(T)) { + if (TII->isTailCall(T) && op.HandleTailcall) { // Treat the tail call as a return instruction, which has a // different-looking sled than the normal return case. Opc = TargetOpcode::PATCHABLE_TAIL_CALL; @@ -108,14 +119,23 @@ } void XRayInstrumentation::prependRetWithPatchableExit( - MachineFunction &MF, const TargetInstrInfo *TII) { + MachineFunction &MF, const TargetInstrInfo *TII, Option op) { for (auto &MBB : MF) - for (auto &T : MBB.terminators()) - if (T.isReturn()) { - // Prepend the return instruction with PATCHABLE_FUNCTION_EXIT. - BuildMI(MBB, T, T.getDebugLoc(), - TII->get(TargetOpcode::PATCHABLE_FUNCTION_EXIT)); + for (auto &T : MBB.terminators()) { + unsigned Opc = 0; + if (T.isReturn() && + (op.HandleAllReturns || T.getOpcode() == TII->getReturnOpcode())) { + Opc = TargetOpcode::PATCHABLE_FUNCTION_EXIT; + } + if (TII->isTailCall(T) && op.HandleTailcall) { + Opc = TargetOpcode::PATCHABLE_TAIL_CALL; } + if (Opc != 0) { + // Prepend the return instruction with PATCHABLE_FUNCTION_EXIT or + // PATCHABLE_TAIL_CALL . + BuildMI(MBB, T, T.getDebugLoc(), TII->get(Opc)); + } + } } bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) { @@ -171,20 +191,34 @@ case Triple::ArchType::arm: case Triple::ArchType::thumb: case Triple::ArchType::aarch64: - case Triple::ArchType::ppc64le: case Triple::ArchType::mips: case Triple::ArchType::mipsel: case Triple::ArchType::mips64: - case Triple::ArchType::mips64el: + case Triple::ArchType::mips64el: { // For the architectures which don't have a single return instruction - prependRetWithPatchableExit(MF, TII); + Option op; + op.HandleTailcall = false; + op.HandleAllReturns = true; + prependRetWithPatchableExit(MF, TII, op); break; - default: + } + case Triple::ArchType::ppc64le: { + Option op; + op.HandleTailcall = false; + op.HandleAllReturns = true; + replaceRetWithPatchableRet(MF, TII, op); + break; + } + default: { // For the architectures that have a single return instruction (such as // RETQ on x86_64). - replaceRetWithPatchableRet(MF, TII); + Option op; + op.HandleTailcall = true; + op.HandleAllReturns = false; + replaceRetWithPatchableRet(MF, TII, op); break; } + } return true; } Index: llvm/lib/Target/PowerPC/PPC.h =================================================================== --- llvm/lib/Target/PowerPC/PPC.h +++ llvm/lib/Target/PowerPC/PPC.h @@ -26,8 +26,10 @@ class PassRegistry; class FunctionPass; class MachineInstr; + class MachineOperand; class AsmPrinter; class MCInst; + class MCOperand; FunctionPass *createPPCCTRLoops(); #ifndef NDEBUG @@ -49,6 +51,9 @@ FunctionPass *createPPCExpandISELPass(); void LowerPPCMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, AsmPrinter &AP, bool isDarwin); + bool LowerPPCMachineOperandToMCOperand(const MachineOperand &MO, + MCOperand &OutMO, AsmPrinter &AP, + bool isDarwin); void initializePPCVSXFMAMutatePass(PassRegistry&); void initializePPCBoolRetToIntPass(PassRegistry&); Index: llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -19,6 +19,7 @@ #include "InstPrinter/PPCInstPrinter.h" #include "MCTargetDesc/PPCMCExpr.h" #include "MCTargetDesc/PPCMCTargetDesc.h" +#include "MCTargetDesc/PPCPredicates.h" #include "PPC.h" #include "PPCInstrInfo.h" #include "PPCMachineFunctionInfo.h" @@ -1089,33 +1090,74 @@ recordSled(BeginOfSled, *MI, SledKind::FUNCTION_ENTER); break; } - case TargetOpcode::PATCHABLE_FUNCTION_EXIT: { - // .p2align 3 - // .begin: - // b(lr)? # lis 0, FuncId[16..32] - // nop # li 0, FuncId[0..15] - // std 0, -8(1) - // mflr 0 - // bl __xray_FunctionExit - // mtlr 0 - // .end: - // b(lr)? - // - // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number - // of instructions change. - const MachineInstr *Next = [&] { - MachineBasicBlock::const_iterator It(MI); - assert(It != MI->getParent()->end()); - ++It; - assert(It->isReturn()); - return &*It; - }(); + case TargetOpcode::PATCHABLE_RET: { + unsigned RetOpcode = MI->getOperand(0).getImm(); + bool IsConditional; + if (RetOpcode == PPC::BCCLR) { + IsConditional = true; + } else if (RetOpcode == PPC::TCRETURNdi8 || RetOpcode == PPC::TCRETURNri8 || + RetOpcode == PPC::TCRETURNai8) { + break; + } else { + assert(RetOpcode == PPC::BLR8 || RetOpcode == PPC::TAILB8); + IsConditional = false; + } + + MCInst RetInst; + MCSymbol *FallthroughLabel = nullptr; + if (IsConditional) { + // Before: + // bgtlr cr0 + // + // After + // ble cr0, .end + // .p2align 3 + // .begin: + // blr # lis 0, FuncId[16..32] + // nop # li 0, FuncId[0..15] + // std 0, -8(1) + // mflr 0 + // bl __xray_FunctionExit + // mtlr 0 + // blr + // .end: + // + // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number + // of instructions change. + FallthroughLabel = OutContext.createTempSymbol(); + EmitToStreamer( + *OutStreamer, + MCInstBuilder(PPC::BCC) + .addImm(PPC::InvertPredicate( + static_cast(MI->getOperand(1).getImm()))) + .addReg(MI->getOperand(2).getReg()) + .addExpr(MCSymbolRefExpr::create(FallthroughLabel, OutContext))); + RetInst.setOpcode(PPC::BLR8); + } else { + // .p2align 3 + // .begin: + // b(lr)? # lis 0, FuncId[16..32] + // nop # li 0, FuncId[0..15] + // std 0, -8(1) + // mflr 0 + // bl __xray_FunctionExit + // mtlr 0 + // b(lr)? + // + // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number + // of instructions change. + RetInst.setOpcode(RetOpcode); + for (const auto &MO : + make_range(std::next(MI->operands_begin()), MI->operands_end())) { + MCOperand MCOp; + if (LowerPPCMachineOperandToMCOperand(MO, MCOp, *this, false)) + RetInst.addOperand(MCOp); + } + } OutStreamer->EmitCodeAlignment(8); MCSymbol *BeginOfSled = OutContext.createTempSymbol(); OutStreamer->EmitLabel(BeginOfSled); - MCInst TmpInst; - LowerPPCMachineInstrToMCInst(Next, TmpInst, *this, false); - EmitToStreamer(*OutStreamer, TmpInst); + EmitToStreamer(*OutStreamer, RetInst); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::NOP)); EmitToStreamer( *OutStreamer, @@ -1127,17 +1169,18 @@ OutContext.getOrCreateSymbol("__xray_FunctionExit"), OutContext))); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTLR8).addReg(PPC::X0)); + EmitToStreamer(*OutStreamer, RetInst); + if (FallthroughLabel) + OutStreamer->EmitLabel(FallthroughLabel); recordSled(BeginOfSled, *MI, SledKind::FUNCTION_EXIT); break; } + case TargetOpcode::PATCHABLE_FUNCTION_EXIT: + llvm_unreachable("PATCHABLE_FUNCTION_EXIT should never be emitted"); case TargetOpcode::PATCHABLE_TAIL_CALL: // TODO: Define a trampoline `__xray_FunctionTailExit` and differentiate a // normal function exit from a tail exit. - case TargetOpcode::PATCHABLE_RET: - // PPC's tail call instruction, e.g. PPC::TCRETURNdi8, doesn't really - // lower to a PPC::B instruction. The PPC::B instruction is generated - // before it, and handled by the normal case. - llvm_unreachable("Tail call is handled in the normal case. See comments" + llvm_unreachable("Tail call is handled in the normal case. See comments " "around this assert."); } } Index: llvm/lib/Target/PowerPC/PPCMCInstLower.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCMCInstLower.cpp +++ llvm/lib/Target/PowerPC/PPCMCInstLower.cpp @@ -143,45 +143,48 @@ OutMI.setOpcode(MI->getOpcode()); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - MCOperand MCOp; - switch (MO.getType()) { - default: - MI->print(errs()); - llvm_unreachable("unknown operand type"); - case MachineOperand::MO_Register: - assert(!MO.getSubReg() && "Subregs should be eliminated!"); - assert(MO.getReg() > PPC::NoRegister && - MO.getReg() < PPC::NUM_TARGET_REGS && - "Invalid register for this target!"); - MCOp = MCOperand::createReg(MO.getReg()); - break; - case MachineOperand::MO_Immediate: - MCOp = MCOperand::createImm(MO.getImm()); - break; - case MachineOperand::MO_MachineBasicBlock: - MCOp = MCOperand::createExpr(MCSymbolRefExpr::create( - MO.getMBB()->getSymbol(), AP.OutContext)); - break; - case MachineOperand::MO_GlobalAddress: - case MachineOperand::MO_ExternalSymbol: - MCOp = GetSymbolRef(MO, GetSymbolFromOperand(MO, AP), AP, isDarwin); - break; - case MachineOperand::MO_JumpTableIndex: - MCOp = GetSymbolRef(MO, AP.GetJTISymbol(MO.getIndex()), AP, isDarwin); - break; - case MachineOperand::MO_ConstantPoolIndex: - MCOp = GetSymbolRef(MO, AP.GetCPISymbol(MO.getIndex()), AP, isDarwin); - break; - case MachineOperand::MO_BlockAddress: - MCOp = GetSymbolRef(MO,AP.GetBlockAddressSymbol(MO.getBlockAddress()),AP, - isDarwin); - break; - case MachineOperand::MO_RegisterMask: - continue; - } - - OutMI.addOperand(MCOp); + if (LowerPPCMachineOperandToMCOperand(MI->getOperand(i), MCOp, AP, + isDarwin)) + OutMI.addOperand(MCOp); + } +} + +bool llvm::LowerPPCMachineOperandToMCOperand(const MachineOperand &MO, + MCOperand &OutMO, AsmPrinter &AP, + bool isDarwin) { + switch (MO.getType()) { + default: + llvm_unreachable("unknown operand type"); + case MachineOperand::MO_Register: + assert(!MO.getSubReg() && "Subregs should be eliminated!"); + assert(MO.getReg() > PPC::NoRegister && + MO.getReg() < PPC::NUM_TARGET_REGS && + "Invalid register for this target!"); + OutMO = MCOperand::createReg(MO.getReg()); + return true; + case MachineOperand::MO_Immediate: + OutMO = MCOperand::createImm(MO.getImm()); + return true; + case MachineOperand::MO_MachineBasicBlock: + OutMO = MCOperand::createExpr( + MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), AP.OutContext)); + return true; + case MachineOperand::MO_GlobalAddress: + case MachineOperand::MO_ExternalSymbol: + OutMO = GetSymbolRef(MO, GetSymbolFromOperand(MO, AP), AP, isDarwin); + return true; + case MachineOperand::MO_JumpTableIndex: + OutMO = GetSymbolRef(MO, AP.GetJTISymbol(MO.getIndex()), AP, isDarwin); + return true; + case MachineOperand::MO_ConstantPoolIndex: + OutMO = GetSymbolRef(MO, AP.GetCPISymbol(MO.getIndex()), AP, isDarwin); + return true; + case MachineOperand::MO_BlockAddress: + OutMO = GetSymbolRef(MO, AP.GetBlockAddressSymbol(MO.getBlockAddress()), AP, + isDarwin); + return true; + case MachineOperand::MO_RegisterMask: + return false; } } Index: llvm/test/CodeGen/PowerPC/xray-conditional-return.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/PowerPC/xray-conditional-return.ll @@ -0,0 +1,79 @@ +; RUN: llc -filetype=asm -o - -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s + +define void @Foo(i32 signext %a, i32 signext %b) #0 { +; CHECK-LABEL: @Foo +; CHECK: cmpw [[CR:[0-9]+]] +; CHECK-NEXT: ble [[CR]], [[LABEL:\.[a-zA-Z0-9]+]] +; CHECK-NEXT: .p2align 3 +; CHECK-NEXT: {{\.[a-zA-Z0-9]+}}: +; CHECK-NEXT: blr +; CHECK-NEXT: nop +; CHECK-NEXT: std 0 +; CHECK-NEXT: mflr 0 +; CHECK-NEXT: bl __xray_FunctionExit +; CHECK-NEXT: nop +; CHECK-NEXT: mtlr 0 +; CHECK-NEXT: blr +; CHECK-NEXT: [[LABEL]]: +entry: + %cmp = icmp sgt i32 %a, %b + br i1 %cmp, label %return, label %if.end + +; CHECK: .p2align 3 +; CHECK-NEXT: {{\.[a-zA-Z0-9]+}}: +; CHECK-NEXT: blr +; CHECK-NEXT: nop +; CHECK-NEXT: std 0 +; CHECK-NEXT: mflr 0 +; CHECK-NEXT: bl __xray_FunctionExit +; CHECK-NEXT: nop +; CHECK-NEXT: mtlr 0 +; CHECK-NEXT: blr +if.end: + tail call void @Bar() + br label %return + +return: + ret void +} + +define void @Foo2(i32 signext %a, i32 signext %b) #0 { +; CHECK-LABEL: @Foo2 +; CHECK: cmpw [[CR:[0-9]+]] +; CHECK-NEXT: bge [[CR]], [[LABEL:\.[a-zA-Z0-9]+]] +; CHECK-NEXT: .p2align 3 +; CHECK-NEXT: {{\.[a-zA-Z0-9]+}}: +; CHECK-NEXT: blr +; CHECK-NEXT: nop +; CHECK-NEXT: std 0 +; CHECK-NEXT: mflr 0 +; CHECK-NEXT: bl __xray_FunctionExit +; CHECK-NEXT: nop +; CHECK-NEXT: mtlr 0 +; CHECK-NEXT: blr +; CHECK-NEXT: [[LABEL]]: +entry: + %cmp = icmp slt i32 %a, %b + br i1 %cmp, label %return, label %if.end + +; CHECK: .p2align 3 +; CHECK-NEXT: {{\.[a-zA-Z0-9]+}}: +; CHECK-NEXT: blr +; CHECK-NEXT: nop +; CHECK-NEXT: std 0 +; CHECK-NEXT: mflr 0 +; CHECK-NEXT: bl __xray_FunctionExit +; CHECK-NEXT: nop +; CHECK-NEXT: mtlr 0 +; CHECK-NEXT: blr +if.end: + tail call void @Bar() + br label %return + +return: + ret void +} + +declare void @Bar() + +attributes #0 = { "function-instrument"="xray-always" }