diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -125,6 +125,7 @@ public: enum class State : char { Empty = 0, /// Function body is empty. + Decoded, /// Instructions have been populated. Disassembled, /// Function have been disassembled. CFG, /// Control flow graph has been built. CFG_Finalized, /// CFG is finalized. No optimizations allowed. @@ -2052,14 +2053,19 @@ /// Disassemble function from raw data. /// If successful, this function will populate the list of instructions /// for this function together with offsets from the function start - /// in the input. It will also populate Labels with destinations for - /// local branches, and TakenBranches with [from, to] info. - /// + /// in the input. /// The Function should be properly initialized before this function /// is called. I.e. function address and size should be set. /// /// Returns true on successful disassembly, and updates the current - /// state to State:Disassembled. + /// state to State::Decoded. + bool decode(MCPlusBuilder::AllocatorIdTy AllocId); + + /// If successful, this function will populate Labels with destinations for + /// local branches, and TakenBranches with [from, to] info. + /// + /// Returns true on successful disassembly, and updates the current + /// state to State::Disassembled. /// /// Returns false if disassembly failed. bool disassemble(); diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -161,6 +161,9 @@ /// Associate profile data with functions and data objects. void processProfileData(); + /// Disassemble instructions in each function in the binary. + void disassembleInstructions(); + /// Disassemble each function in the binary and associate it with a /// BinaryFunction object, preparing all information necessary for binary /// optimization. diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -217,6 +217,7 @@ raw_ostream &operator<<(raw_ostream &OS, const BinaryFunction::State State) { switch (State) { case BinaryFunction::State::Empty: OS << "empty"; break; + case BinaryFunction::State::Decoded: OS << "decoded"; break; case BinaryFunction::State::Disassembled: OS << "disassembled"; break; case BinaryFunction::State::CFG: OS << "CFG constructed"; break; case BinaryFunction::State::CFG_Finalized: OS << "CFG finalized"; break; @@ -1159,7 +1160,7 @@ } } -bool BinaryFunction::disassemble() { +bool BinaryFunction::decode(MCPlusBuilder::AllocatorIdTy AllocId) { NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs", "Build Binary Functions", opts::TimeBuild); ErrorOr> ErrorOrFunctionData = getData(); @@ -1171,11 +1172,10 @@ auto &Ctx = BC.Ctx; auto &MIB = BC.MIB; - BC.SymbolicDisAsm->setSymbolizer(MIB->createTargetSymbolizer(*this)); - - // Insert a label at the beginning of the function. This will be our first - // basic block. - Labels[0] = Ctx->createNamedTempSymbol("BB0"); + MCDisassembler *SymbolicDisAsm( + BC.TheTarget->createMCDisassembler(*BC.STI, *Ctx)); + assert(SymbolicDisAsm); + SymbolicDisAsm->setSymbolizer(MIB->createTargetSymbolizer(*this)); uint64_t Size = 0; // instruction size for (uint64_t Offset = 0; Offset < getSize(); Offset += Size) { @@ -1188,14 +1188,15 @@ continue; } - if (!BC.SymbolicDisAsm->getInstruction(Instruction, Size, - FunctionData.slice(Offset), - AbsoluteInstrAddr, nulls())) { + if (!SymbolicDisAsm->getInstruction(Instruction, Size, + FunctionData.slice(Offset), + AbsoluteInstrAddr, nulls())) { // Functions with "soft" boundaries, e.g. coming from assembly source, // can have 0-byte padding at the end. if (isZeroPaddingAt(Offset)) break; + auto L = BC.scopeLock(); errs() << "BOLT-WARNING: unable to disassemble instruction at offset 0x" << Twine::utohexstr(Offset) << " (address 0x" << Twine::utohexstr(AbsoluteInstrAddr) << ") in function " << *this @@ -1215,6 +1216,7 @@ if (opts::CheckEncoding && !BC.MIB->isBranch(Instruction) && !BC.MIB->isCall(Instruction) && !BC.MIB->isNoop(Instruction)) { if (!BC.validateInstructionEncoding(FunctionData.slice(Offset, Size))) { + auto L = BC.scopeLock(); errs() << "BOLT-WARNING: mismatching LLVM encoding detected in " << "function " << *this << " for instruction :\n"; BC.printInstruction(errs(), Instruction, AbsoluteInstrAddr); @@ -1225,12 +1227,14 @@ // Special handling for AVX-512 instructions. if (MIB->hasEVEXEncoding(Instruction)) { if (BC.HasRelocations && opts::TrapOnAVX512) { + auto L = BC.scopeLock(); setTrapOnEntry(); BC.TrappedFunctions.push_back(this); break; } if (!BC.validateInstructionEncoding(FunctionData.slice(Offset, Size))) { + auto L = BC.scopeLock(); errs() << "BOLT-WARNING: internal assembler/disassembler error " "detected for AVX512 instruction:\n"; BC.printInstruction(errs(), Instruction, AbsoluteInstrAddr); @@ -1240,6 +1244,60 @@ } } + // add instruction + if (getDWARFLineTable()) { + Instruction.setLoc(findDebugLineInformationForInstructionAt( + AbsoluteInstrAddr, getDWARFUnit(), getDWARFLineTable())); + } + + // Record offset of the instruction for profile matching. + if (BC.keepOffsetForInstruction(Instruction)) + MIB->setOffset(Instruction, static_cast(Offset), AllocId); + + // NOTE: disassembly loses the correct size information for noops. + // E.g. nopw 0x0(%rax,%rax,1) is 9 bytes, but re-encoded it's only + // 5 bytes. Preserve the size info using annotations. + MIB->addAnnotation(Instruction, "Size", static_cast(Size), + AllocId); + + addInstruction(Offset, std::move(Instruction)); + } + + // Reset symbolizer for the disassembler. + SymbolicDisAsm->setSymbolizer(nullptr); + + updateState(State::Decoded); + + return IsSimple; +} + +bool BinaryFunction::disassemble() { + NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs", + "Build Binary Functions", opts::TimeBuild); + ErrorOr> ErrorOrFunctionData = getData(); + assert(ErrorOrFunctionData && "function data is not available"); + ArrayRef FunctionData = *ErrorOrFunctionData; + assert(FunctionData.size() == getMaxSize() && + "function size does not match raw data size"); + + auto &Ctx = BC.Ctx; + auto &MIB = BC.MIB; + + // Insert a label at the beginning of the function. This will be our first + // basic block. + Labels[0] = Ctx->createNamedTempSymbol("BB0"); + + for (auto &InstructionOffsetPair : Instructions) { + uint64_t Offset = InstructionOffsetPair.first; + MCInst &Instruction = InstructionOffsetPair.second; + const uint64_t AbsoluteInstrAddr = getAddress() + Offset; + // instruction size + uint32_t Size = MIB->getAnnotationWithDefault(Instruction, "Size", 0); + assert(Size); + // Only preserve Size annotation for nops. + if (!BC.MIB->isNoop(Instruction)) + MIB->removeAnnotation(Instruction, "Size"); + if (MIB->isBranch(Instruction) || MIB->isCall(Instruction)) { uint64_t TargetAddress = 0; if (MIB->evaluateBranch(Instruction, AbsoluteInstrAddr, Size, @@ -1297,7 +1355,7 @@ // Register branch offset for profile validation. IgnoredBranches.emplace_back(Offset, Offset + Size); } - goto add_instruction; + continue; } // May update Instruction and IsCall TargetSymbol = handleExternalReference(Instruction, Size, Offset, @@ -1349,30 +1407,8 @@ if (MIB->hasPCRelOperand(Instruction) && !UsedReloc) handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size); } - -add_instruction: - if (getDWARFLineTable()) { - Instruction.setLoc(findDebugLineInformationForInstructionAt( - AbsoluteInstrAddr, getDWARFUnit(), getDWARFLineTable())); - } - - // Record offset of the instruction for profile matching. - if (BC.keepOffsetForInstruction(Instruction)) - MIB->setOffset(Instruction, static_cast(Offset)); - - if (BC.MIB->isNoop(Instruction)) { - // NOTE: disassembly loses the correct size information for noops. - // E.g. nopw 0x0(%rax,%rax,1) is 9 bytes, but re-encoded it's only - // 5 bytes. Preserve the size info using annotations. - MIB->addAnnotation(Instruction, "Size", static_cast(Size)); - } - - addInstruction(Offset, std::move(Instruction)); } - // Reset symbolizer for the disassembler. - BC.SymbolicDisAsm->setSymbolizer(nullptr); - if (uint64_t Offset = getFirstInstructionOffset()) Labels[Offset] = BC.Ctx->createNamedTempSymbol(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -746,6 +746,8 @@ readDebugInfo(); + disassembleInstructions(); + disassembleFunctions(); processProfileDataPreCFG(); @@ -2905,6 +2907,65 @@ exit(0); } +void RewriteInstance::disassembleInstructions() { + NamedRegionTimer T("disassembleInstructions", "disassemble instructions", + TimerGroupName, TimerGroupDesc, opts::TimeRewrite); + + ParallelUtilities::WorkFuncWithAllocTy DisassembleFun = + [&](BinaryFunction &Function, MCPlusBuilder::AllocatorIdTy AllocId) { + ErrorOr> FunctionData = Function.getData(); + if (!FunctionData) { + auto L = BC->scopeLock(); + errs() << "BOLT-ERROR: corresponding section is non-executable or " + << "empty for function " << Function << '\n'; + exit(1); + } + + // Treat zero-sized functions as non-simple ones. + if (Function.getSize() == 0) { + Function.setSimple(false); + return; + } + + // Offset of the function in the file. + const auto *FileBegin = + reinterpret_cast(InputFile->getData().data()); + Function.setFileOffset(FunctionData->begin() - FileBegin); + + if (!shouldDisassemble(Function)) { + NamedRegionTimer T("scan", "scan functions", "buildfuncs", + "Scan Binary Functions", opts::TimeBuild); + Function.scanExternalRefs(); + Function.setSimple(false); + return; + } + + if (!Function.decode(AllocId)) { + if (opts::processAllFunctions()) + BC->exitWithBugReport("function cannot be properly disassembled. " + "Unable to continue in relocation mode.", + Function); + if (opts::Verbosity >= 1) + outs() << "BOLT-INFO: could not disassemble function " << Function + << ". Will ignore.\n"; + // Forcefully ignore the function. + Function.setIgnored(); + return; + } + + if (opts::PrintAll || opts::PrintDisasm) { + auto L = BC->scopeLock(); + Function.print(outs(), "after disassembly"); + } + }; + + ParallelUtilities::runOnEachFunctionWithUniqueAllocId( + *BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, DisassembleFun, + [](const BinaryFunction &BF) { return false; }, + "disassembleFunctions-disassemble", + /*ForceSequential*/ opts::SequentialDisassembly || opts::PrintAll); +} + void RewriteInstance::disassembleFunctions() { NamedRegionTimer T("disassembleFunctions", "disassemble functions", TimerGroupName, TimerGroupDesc, opts::TimeRewrite); diff --git a/bolt/lib/Target/X86/X86MCSymbolizer.cpp b/bolt/lib/Target/X86/X86MCSymbolizer.cpp --- a/bolt/lib/Target/X86/X86MCSymbolizer.cpp +++ b/bolt/lib/Target/X86/X86MCSymbolizer.cpp @@ -70,6 +70,7 @@ const MCSymbol *TargetSymbol; uint64_t TargetOffset; + auto L = BC.scopeLock(); std::tie(TargetSymbol, TargetOffset) = BC.handleAddressRef(Value, Function, /*IsPCRel=*/true); @@ -98,6 +99,7 @@ // The linker converted the PC-relative address to an absolute one. // Symbolize this address. + auto L = BC.scopeLock(); BC.handleAddressRef(Value, Function, /*IsPCRel=*/false); const BinaryData *Target = BC.getBinaryDataAtAddress(Value); assert(Target && @@ -119,6 +121,7 @@ if (Relocation->isPCRelative()) SymbolValue += InstAddress + ImmOffset; + auto L = BC.scopeLock(); // Process reference to the symbol. BC.handleAddressRef(SymbolValue, Function, Relocation->isPCRelative());