Index: llvm/lib/Target/X86/X86IndirectThunks.cpp =================================================================== --- llvm/lib/Target/X86/X86IndirectThunks.cpp +++ llvm/lib/Target/X86/X86IndirectThunks.cpp @@ -48,6 +48,35 @@ static const char EDIRetpolineName[] = "__llvm_retpoline_edi"; namespace { +template class ThunkInserter { + Derived &getDerived() { return *static_cast(this); } + +protected: + bool InsertedThunks; + void doInitialization(Module &M) {} + void createThunkFunction(MachineModuleInfo &MMI, StringRef Name); + +public: + void init(Module &M) { + InsertedThunks = false; + getDerived().doInitialization(M); + } + // return `true` if `MMI` or `MF` was modified + bool run(MachineModuleInfo &MMI, MachineFunction &MF); +}; + +struct RetpolineThunkInserter : ThunkInserter { + const char *getThunkPrefix() { return RetpolineNamePrefix; } + bool mayUseThunk(const MachineFunction &MF) { + const auto &STI = MF.getSubtarget(); + return (STI.useRetpolineIndirectCalls() || + STI.useRetpolineIndirectBranches()) && + !STI.useRetpolineExternalThunk(); + } + void insertThunks(MachineModuleInfo &MMI); + void populateThunk(MachineFunction &MF); +}; + class X86IndirectThunks : public MachineFunctionPass { public: static char ID; @@ -57,7 +86,7 @@ StringRef getPassName() const override { return "X86 Indirect Thunks"; } bool doInitialization(Module &M) override; - bool runOnMachineFunction(MachineFunction &F) override; + bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { MachineFunctionPass::getAnalysisUsage(AU); @@ -66,78 +95,39 @@ } private: - MachineModuleInfo *MMI = nullptr; - const TargetMachine *TM = nullptr; - bool Is64Bit = false; - const X86Subtarget *STI = nullptr; - const X86InstrInfo *TII = nullptr; - - bool InsertedThunks = false; - - void createThunkFunction(Module &M, StringRef Name); - void insertRegReturnAddrClobber(MachineBasicBlock &MBB, Register Reg); - void populateThunk(MachineFunction &MF, Register Reg); + std::tuple TIs; + + // FIXME: When LLVM moves to C++17, these can become folds + template + static void initTIs(Module &M, + std::tuple &ThunkInserters) { + (void)std::initializer_list{ + (std::get(ThunkInserters).init(M), 0)...}; + } + template + static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF, + std::tuple &ThunkInserters) { + bool Modified = false; + (void)std::initializer_list{ + Modified |= std::get(ThunkInserters).run(MMI, MF)...}; + return Modified; + } }; } // end anonymous namespace -FunctionPass *llvm::createX86IndirectThunksPass() { - return new X86IndirectThunks(); +void RetpolineThunkInserter::insertThunks(MachineModuleInfo &MMI) { + if (MMI.getTarget().getTargetTriple().getArch() == Triple::x86_64) + createThunkFunction(MMI, R11RetpolineName); + else + for (StringRef Name : {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName, + EDIRetpolineName}) + createThunkFunction(MMI, Name); } -char X86IndirectThunks::ID = 0; - -bool X86IndirectThunks::doInitialization(Module &M) { - InsertedThunks = false; - return false; -} - -bool X86IndirectThunks::runOnMachineFunction(MachineFunction &MF) { - LLVM_DEBUG(dbgs() << getPassName() << '\n'); - - TM = &MF.getTarget();; - STI = &MF.getSubtarget(); - TII = STI->getInstrInfo(); - Is64Bit = TM->getTargetTriple().getArch() == Triple::x86_64; - - MMI = &getAnalysis().getMMI(); - Module &M = const_cast(*MMI->getModule()); - - // If this function is not a thunk, check to see if we need to insert - // a thunk. - if (!MF.getName().startswith(RetpolineNamePrefix)) { - // If we've already inserted a thunk, nothing else to do. - if (InsertedThunks) - return false; - - // Only add a thunk if one of the functions has the retpoline feature - // enabled in its subtarget, and doesn't enable external thunks. - // FIXME: Conditionalize on indirect calls so we don't emit a thunk when - // nothing will end up calling it. - // FIXME: It's a little silly to look at every function just to enumerate - // the subtargets, but eventually we'll want to look at them for indirect - // calls, so maybe this is OK. - if ((!STI->useRetpolineIndirectCalls() && - !STI->useRetpolineIndirectBranches()) || - STI->useRetpolineExternalThunk()) - return false; - - // Otherwise, we need to insert the thunk. - // WARNING: This is not really a well behaving thing to do in a function - // pass. We extract the module and insert a new function (and machine - // function) directly into the module. - if (Is64Bit) - createThunkFunction(M, R11RetpolineName); - else - for (StringRef Name : - {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName, - EDIRetpolineName}) - createThunkFunction(M, Name); - InsertedThunks = true; - return true; - } - - // If this *is* a thunk function, we need to populate it with the correct MI. +void RetpolineThunkInserter::populateThunk(MachineFunction &MF) { + bool Is64Bit = MF.getTarget().getTargetTriple().getArch() == Triple::x86_64; + Register ThunkReg; if (Is64Bit) { assert(MF.getName() == "__llvm_retpoline_r11" && "Should only have an r11 thunk on 64-bit targets"); @@ -152,7 +142,7 @@ // .Lr11_call_target: // movq %r11, (%rsp) // retq - populateThunk(MF, X86::R11); + ThunkReg = X86::R11; } else { // For 32-bit targets we need to emit a collection of thunks for various // possible scratch registers as well as a fallback that uses EDI, which is @@ -182,67 +172,18 @@ // movl %edi, (%esp) // retl if (MF.getName() == EAXRetpolineName) - populateThunk(MF, X86::EAX); + ThunkReg = X86::EAX; else if (MF.getName() == ECXRetpolineName) - populateThunk(MF, X86::ECX); + ThunkReg = X86::ECX; else if (MF.getName() == EDXRetpolineName) - populateThunk(MF, X86::EDX); + ThunkReg = X86::EDX; else if (MF.getName() == EDIRetpolineName) - populateThunk(MF, X86::EDI); + ThunkReg = X86::EDI; else llvm_unreachable("Invalid thunk name on x86-32!"); } - return true; -} - -void X86IndirectThunks::createThunkFunction(Module &M, StringRef Name) { - assert(Name.startswith(RetpolineNamePrefix) && - "Created a thunk with an unexpected prefix!"); - - LLVMContext &Ctx = M.getContext(); - auto Type = FunctionType::get(Type::getVoidTy(Ctx), false); - Function *F = - Function::Create(Type, GlobalValue::LinkOnceODRLinkage, Name, &M); - F->setVisibility(GlobalValue::HiddenVisibility); - F->setComdat(M.getOrInsertComdat(Name)); - - // Add Attributes so that we don't create a frame, unwind information, or - // inline. - AttrBuilder B; - B.addAttribute(llvm::Attribute::NoUnwind); - B.addAttribute(llvm::Attribute::Naked); - F->addAttributes(llvm::AttributeList::FunctionIndex, B); - - // Populate our function a bit so that we can verify. - BasicBlock *Entry = BasicBlock::Create(Ctx, "entry", F); - IRBuilder<> Builder(Entry); - - Builder.CreateRetVoid(); - - // MachineFunctions/MachineBasicBlocks aren't created automatically for the - // IR-level constructs we already made. Create them and insert them into the - // module. - MachineFunction &MF = MMI->getOrCreateMachineFunction(*F); - MachineBasicBlock *EntryMBB = MF.CreateMachineBasicBlock(Entry); - - // Insert EntryMBB into MF. It's not in the module until we do this. - MF.insert(MF.end(), EntryMBB); -} - -void X86IndirectThunks::insertRegReturnAddrClobber(MachineBasicBlock &MBB, - Register Reg) { - const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr; - const Register SPReg = Is64Bit ? X86::RSP : X86::ESP; - addRegOffset(BuildMI(&MBB, DebugLoc(), TII->get(MovOpc)), SPReg, false, 0) - .addReg(Reg); -} - -void X86IndirectThunks::populateThunk(MachineFunction &MF, - Register Reg) { - // Set MF properties. We never use vregs... - MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs); - + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); // Grab the entry MBB and erase any other blocks. O0 codegen appears to // generate two bbs for the entry block. MachineBasicBlock *Entry = &MF.front(); @@ -261,7 +202,7 @@ const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32; const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL; - Entry->addLiveIn(Reg); + Entry->addLiveIn(ThunkReg); BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym); // The MIR verifier thinks that the CALL in the entry block will fall through @@ -283,10 +224,101 @@ CaptureSpec->setHasAddressTaken(); CaptureSpec->addSuccessor(CaptureSpec); - CallTarget->addLiveIn(Reg); + CallTarget->addLiveIn(ThunkReg); CallTarget->setHasAddressTaken(); CallTarget->setAlignment(Align(16)); - insertRegReturnAddrClobber(*CallTarget, Reg); + + // Insert return address clobber + const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr; + const Register SPReg = Is64Bit ? X86::RSP : X86::ESP; + addRegOffset(BuildMI(CallTarget, DebugLoc(), TII->get(MovOpc)), SPReg, false, + 0) + .addReg(ThunkReg); + CallTarget->back().setPreInstrSymbol(MF, TargetSym); BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc)); } + +template +void ThunkInserter::createThunkFunction(MachineModuleInfo &MMI, + StringRef Name) { + assert(Name.startswith(getDerived().getThunkPrefix()) && + "Created a thunk with an unexpected prefix!"); + + Module &M = const_cast(*MMI.getModule()); + LLVMContext &Ctx = M.getContext(); + auto Type = FunctionType::get(Type::getVoidTy(Ctx), false); + Function *F = + Function::Create(Type, GlobalValue::LinkOnceODRLinkage, Name, &M); + F->setVisibility(GlobalValue::HiddenVisibility); + F->setComdat(M.getOrInsertComdat(Name)); + + // Add Attributes so that we don't create a frame, unwind information, or + // inline. + AttrBuilder B; + B.addAttribute(llvm::Attribute::NoUnwind); + B.addAttribute(llvm::Attribute::Naked); + F->addAttributes(llvm::AttributeList::FunctionIndex, B); + + // Populate our function a bit so that we can verify. + BasicBlock *Entry = BasicBlock::Create(Ctx, "entry", F); + IRBuilder<> Builder(Entry); + + Builder.CreateRetVoid(); + + // MachineFunctions/MachineBasicBlocks aren't created automatically for the + // IR-level constructs we already made. Create them and insert them into the + // module. + MachineFunction &MF = MMI.getOrCreateMachineFunction(*F); + MachineBasicBlock *EntryMBB = MF.CreateMachineBasicBlock(Entry); + + // Insert EntryMBB into MF. It's not in the module until we do this. + MF.insert(MF.end(), EntryMBB); + // Set MF properties. We never use vregs... + MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs); +} + +template +bool ThunkInserter::run(MachineModuleInfo &MMI, MachineFunction &MF) { + // If MF is not a thunk, check to see if we need to insert a thunk. + if (!MF.getName().startswith(getDerived().getThunkPrefix())) { + // If we've already inserted a thunk, nothing else to do. + if (InsertedThunks) + return false; + + // Only add a thunk if one of the functions has the corresponding feature + // enabled in its subtarget, and doesn't enable external thunks. + // FIXME: Conditionalize on indirect calls so we don't emit a thunk when + // nothing will end up calling it. + // FIXME: It's a little silly to look at every function just to enumerate + // the subtargets, but eventually we'll want to look at them for indirect + // calls, so maybe this is OK. + if (!getDerived().mayUseThunk(MF)) + return false; + + getDerived().insertThunks(MMI); + InsertedThunks = true; + return true; + } + + // If this *is* a thunk function, we need to populate it with the correct MI. + getDerived().populateThunk(MF); + return true; +} + +FunctionPass *llvm::createX86IndirectThunksPass() { + return new X86IndirectThunks(); +} + +char X86IndirectThunks::ID = 0; + +bool X86IndirectThunks::doInitialization(Module &M) { + initTIs(M, TIs); + return false; +} + +bool X86IndirectThunks::runOnMachineFunction(MachineFunction &MF) { + LLVM_DEBUG(dbgs() << getPassName() << '\n'); + auto &MMI = getAnalysis().getMMI(); + return runTIs(MMI, MF, TIs); +}