diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -161,6 +161,7 @@ ///< compile step. CODEGENOPT(LTOUnit, 1, 0) ///< Emit IR to support LTO unit features (CFI, whole ///< program vtable opt). +CODEGENOPT(FatLTO, 1, 0) ///< Set when -ffat-lto-objects is enabled. CODEGENOPT(EnableSplitLTOUnit, 1, 0) ///< Enable LTO unit splitting to support /// CFI and traditional whole program /// devirtualization that require whole diff --git a/clang/include/clang/CodeGen/BackendUtil.h b/clang/include/clang/CodeGen/BackendUtil.h --- a/clang/include/clang/CodeGen/BackendUtil.h +++ b/clang/include/clang/CodeGen/BackendUtil.h @@ -45,6 +45,9 @@ void EmbedBitcode(llvm::Module *M, const CodeGenOptions &CGOpts, llvm::MemoryBufferRef Buf); + void EmbedFatLTO(llvm::Module *M, const CodeGenOptions &CGOpts, + llvm::MemoryBufferRef Buf); + void EmbedObject(llvm::Module *M, const CodeGenOptions &CGOpts, DiagnosticsEngine &Diags); } diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2124,6 +2124,11 @@ Alias, AliasArgs<["full"]>, HelpText<"Enable LTO in 'full' mode">; def flto : Flag<["-"], "flto">, Flags<[CoreOption, CC1Option]>, Group, Alias, AliasArgs<["full"]>, HelpText<"Enable LTO in 'full' mode">; +defm fat_lto_objects : BoolFOption<"fat-lto-objects", + CodeGenOpts<"FatLTO">, DefaultFalse, + PosFlag, + NegFlag, + BothFlags<[CC1Option], "">>; def fno_lto : Flag<["-"], "fno-lto">, Flags<[CoreOption, CC1Option]>, Group, HelpText<"Disable LTO mode (default)">; def foffload_lto_EQ : Joined<["-"], "foffload-lto=">, Flags<[CoreOption]>, Group, @@ -4669,7 +4674,6 @@ defm reorder_blocks : BooleanFFlag<"reorder-blocks">, Group; defm branch_count_reg : BooleanFFlag<"branch-count-reg">, Group; defm default_inline : BooleanFFlag<"default-inline">, Group; -defm fat_lto_objects : BooleanFFlag<"fat-lto-objects">, Group; defm float_store : BooleanFFlag<"float-store">, Group; defm friend_injection : BooleanFFlag<"friend-injection">, Group; defm function_attribute_list : BooleanFFlag<"function-attribute-list">, Group; diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h --- a/clang/include/clang/Driver/ToolChain.h +++ b/clang/include/clang/Driver/ToolChain.h @@ -569,6 +569,9 @@ /// SupportsEmbeddedBitcode - Does this tool chain support embedded bitcode. virtual bool SupportsEmbeddedBitcode() const { return false; } + /// SupportsFatLTO - Does this tool chain support fat LTO objects. + virtual bool SupportsFatLTO() const { return true; } + /// getThreadModel() - Which thread model does this target use? virtual std::string getThreadModel() const { return "posix"; } diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -193,7 +193,7 @@ void EmitAssembly(BackendAction Action, std::unique_ptr OS); }; -} +} // namespace static SanitizerCoverageOptions getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) { @@ -1197,6 +1197,8 @@ // __LLVM,__bitcode section. void clang::EmbedBitcode(llvm::Module *M, const CodeGenOptions &CGOpts, llvm::MemoryBufferRef Buf) { + if (CGOpts.FatLTO) + return; if (CGOpts.getEmbedBitcode() == CodeGenOptions::Embed_Off) return; llvm::embedBitcodeInModule( @@ -1205,6 +1207,15 @@ CGOpts.CmdArgs); } +// With -ffat-lto-objects, save a copy of the llvm IR as data in the +// .fatlto section. +void clang::EmbedFatLTO(llvm::Module *M, const CodeGenOptions &CGOpts, + llvm::MemoryBufferRef Buf) { + if (!CGOpts.FatLTO) + return; + llvm::embedBitcodeInFatObject(*M, Buf); +} + void clang::EmbedObject(llvm::Module *M, const CodeGenOptions &CGOpts, DiagnosticsEngine &Diags) { if (CGOpts.OffloadObjects.empty()) diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp --- a/clang/lib/CodeGen/CodeGenAction.cpp +++ b/clang/lib/CodeGen/CodeGenAction.cpp @@ -375,6 +375,7 @@ } EmbedBitcode(getModule(), CodeGenOpts, llvm::MemoryBufferRef()); + EmbedFatLTO(getModule(), CodeGenOpts, llvm::MemoryBufferRef()); EmitBackendOutput(Diags, HeaderSearchOpts, CodeGenOpts, TargetOpts, LangOpts, C.getTargetInfo().getDataLayoutString(), @@ -1172,6 +1173,8 @@ EmbedObject(TheModule.get(), CodeGenOpts, Diagnostics); EmbedBitcode(TheModule.get(), CodeGenOpts, *MainFile); + EmbedFatLTO(TheModule.get(), CodeGenOpts, *MainFile); + LLVMContext &Ctx = TheModule->getContext(); diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -1,5 +1,4 @@ //===--- Driver.cpp - Clang GCC Compatible Driver -------------------------===// -// // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -1314,6 +1313,11 @@ BitcodeEmbed = static_cast(Model); } + // Process -ffat-lto-objects flag. + if (Arg *A = Args.getLastArg(options::OPT_ffat_lto_objects)) { + BitcodeEmbed = EmbedBitcode; + } + // Remove existing compilation database so that each job can append to it. if (Arg *A = Args.getLastArg(options::OPT_MJ)) llvm::sys::fs::remove(A->getValue()); diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -7117,6 +7117,13 @@ } } + // if (IsUsingLTO && LTOMode == LTOK_Full && Args.getLastArg(options::OPT_ffat_lto_objects)) + // CmdArgs.push_back("-ffat-lto-objects"); + + if (Args.hasArg(options::OPT_ffat_lto_objects)) + CmdArgs.push_back("-ffat-lto-objects"); + + if (Args.hasArg(options::OPT_forder_file_instrumentation)) { CmdArgs.push_back("-forder-file-instrumentation"); // Enable order file instrumentation when ThinLTO is not on. When ThinLTO is diff --git a/llvm/include/llvm/Bitcode/BitcodeWriter.h b/llvm/include/llvm/Bitcode/BitcodeWriter.h --- a/llvm/include/llvm/Bitcode/BitcodeWriter.h +++ b/llvm/include/llvm/Bitcode/BitcodeWriter.h @@ -165,6 +165,7 @@ bool EmbedCmdline, const std::vector &CmdArgs); + void embedBitcodeInFatObject(Module &M, MemoryBufferRef Buf); } // end namespace llvm #endif // LLVM_BITCODE_BITCODEWRITER_H diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -351,8 +351,8 @@ unsigned Abbrev); void writeDILocalVariable(const DILocalVariable *N, SmallVectorImpl &Record, unsigned Abbrev); - void writeDILabel(const DILabel *N, - SmallVectorImpl &Record, unsigned Abbrev); + void writeDILabel(const DILabel *N, SmallVectorImpl &Record, + unsigned Abbrev); void writeDIExpression(const DIExpression *N, SmallVectorImpl &Record, unsigned Abbrev); void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N, @@ -402,9 +402,7 @@ void writeBlockInfo(); void writeModuleHash(size_t BlockStartPos); - unsigned getEncodedSyncScopeID(SyncScope::ID SSID) { - return unsigned(SSID); - } + unsigned getEncodedSyncScopeID(SyncScope::ID SSID) { return unsigned(SSID); } unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(Alignment); } }; @@ -439,9 +437,8 @@ // in writing out the call graph edges. Save the mapping from GUID // to the new global value id to use when writing those edges, which // are currently saved in the index in terms of GUID. - forEachSummary([&](GVInfo I, bool) { - GUIDToValueIdMap[I.first] = ++GlobalValueId; - }); + forEachSummary( + [&](GVInfo I, bool) { GUIDToValueIdMap[I.first] = ++GlobalValueId; }); } /// The below iterator returns the GUID and associated summary. @@ -450,8 +447,7 @@ /// Calls the callback for each value GUID and summary to be written to /// bitcode. This hides the details of whether they are being pulled from the /// entire index or just those in a provided ModuleToSummariesForIndex map. - template - void forEachSummary(Functor Callback) { + template void forEachSummary(Functor Callback) { if (ModuleToSummariesForIndex) { for (auto &M : *ModuleToSummariesForIndex) for (auto &Summary : M.second) { @@ -513,82 +509,133 @@ static unsigned getEncodedCastOpcode(unsigned Opcode) { switch (Opcode) { - default: llvm_unreachable("Unknown cast instruction!"); - case Instruction::Trunc : return bitc::CAST_TRUNC; - case Instruction::ZExt : return bitc::CAST_ZEXT; - case Instruction::SExt : return bitc::CAST_SEXT; - case Instruction::FPToUI : return bitc::CAST_FPTOUI; - case Instruction::FPToSI : return bitc::CAST_FPTOSI; - case Instruction::UIToFP : return bitc::CAST_UITOFP; - case Instruction::SIToFP : return bitc::CAST_SITOFP; - case Instruction::FPTrunc : return bitc::CAST_FPTRUNC; - case Instruction::FPExt : return bitc::CAST_FPEXT; - case Instruction::PtrToInt: return bitc::CAST_PTRTOINT; - case Instruction::IntToPtr: return bitc::CAST_INTTOPTR; - case Instruction::BitCast : return bitc::CAST_BITCAST; - case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST; + default: + llvm_unreachable("Unknown cast instruction!"); + case Instruction::Trunc: + return bitc::CAST_TRUNC; + case Instruction::ZExt: + return bitc::CAST_ZEXT; + case Instruction::SExt: + return bitc::CAST_SEXT; + case Instruction::FPToUI: + return bitc::CAST_FPTOUI; + case Instruction::FPToSI: + return bitc::CAST_FPTOSI; + case Instruction::UIToFP: + return bitc::CAST_UITOFP; + case Instruction::SIToFP: + return bitc::CAST_SITOFP; + case Instruction::FPTrunc: + return bitc::CAST_FPTRUNC; + case Instruction::FPExt: + return bitc::CAST_FPEXT; + case Instruction::PtrToInt: + return bitc::CAST_PTRTOINT; + case Instruction::IntToPtr: + return bitc::CAST_INTTOPTR; + case Instruction::BitCast: + return bitc::CAST_BITCAST; + case Instruction::AddrSpaceCast: + return bitc::CAST_ADDRSPACECAST; } } static unsigned getEncodedUnaryOpcode(unsigned Opcode) { switch (Opcode) { - default: llvm_unreachable("Unknown binary instruction!"); - case Instruction::FNeg: return bitc::UNOP_FNEG; + default: + llvm_unreachable("Unknown binary instruction!"); + case Instruction::FNeg: + return bitc::UNOP_FNEG; } } static unsigned getEncodedBinaryOpcode(unsigned Opcode) { switch (Opcode) { - default: llvm_unreachable("Unknown binary instruction!"); + default: + llvm_unreachable("Unknown binary instruction!"); case Instruction::Add: - case Instruction::FAdd: return bitc::BINOP_ADD; + case Instruction::FAdd: + return bitc::BINOP_ADD; case Instruction::Sub: - case Instruction::FSub: return bitc::BINOP_SUB; + case Instruction::FSub: + return bitc::BINOP_SUB; case Instruction::Mul: - case Instruction::FMul: return bitc::BINOP_MUL; - case Instruction::UDiv: return bitc::BINOP_UDIV; + case Instruction::FMul: + return bitc::BINOP_MUL; + case Instruction::UDiv: + return bitc::BINOP_UDIV; case Instruction::FDiv: - case Instruction::SDiv: return bitc::BINOP_SDIV; - case Instruction::URem: return bitc::BINOP_UREM; + case Instruction::SDiv: + return bitc::BINOP_SDIV; + case Instruction::URem: + return bitc::BINOP_UREM; case Instruction::FRem: - case Instruction::SRem: return bitc::BINOP_SREM; - case Instruction::Shl: return bitc::BINOP_SHL; - case Instruction::LShr: return bitc::BINOP_LSHR; - case Instruction::AShr: return bitc::BINOP_ASHR; - case Instruction::And: return bitc::BINOP_AND; - case Instruction::Or: return bitc::BINOP_OR; - case Instruction::Xor: return bitc::BINOP_XOR; + case Instruction::SRem: + return bitc::BINOP_SREM; + case Instruction::Shl: + return bitc::BINOP_SHL; + case Instruction::LShr: + return bitc::BINOP_LSHR; + case Instruction::AShr: + return bitc::BINOP_ASHR; + case Instruction::And: + return bitc::BINOP_AND; + case Instruction::Or: + return bitc::BINOP_OR; + case Instruction::Xor: + return bitc::BINOP_XOR; } } static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) { switch (Op) { - default: llvm_unreachable("Unknown RMW operation!"); - case AtomicRMWInst::Xchg: return bitc::RMW_XCHG; - case AtomicRMWInst::Add: return bitc::RMW_ADD; - case AtomicRMWInst::Sub: return bitc::RMW_SUB; - case AtomicRMWInst::And: return bitc::RMW_AND; - case AtomicRMWInst::Nand: return bitc::RMW_NAND; - case AtomicRMWInst::Or: return bitc::RMW_OR; - case AtomicRMWInst::Xor: return bitc::RMW_XOR; - case AtomicRMWInst::Max: return bitc::RMW_MAX; - case AtomicRMWInst::Min: return bitc::RMW_MIN; - case AtomicRMWInst::UMax: return bitc::RMW_UMAX; - case AtomicRMWInst::UMin: return bitc::RMW_UMIN; - case AtomicRMWInst::FAdd: return bitc::RMW_FADD; - case AtomicRMWInst::FSub: return bitc::RMW_FSUB; + default: + llvm_unreachable("Unknown RMW operation!"); + case AtomicRMWInst::Xchg: + return bitc::RMW_XCHG; + case AtomicRMWInst::Add: + return bitc::RMW_ADD; + case AtomicRMWInst::Sub: + return bitc::RMW_SUB; + case AtomicRMWInst::And: + return bitc::RMW_AND; + case AtomicRMWInst::Nand: + return bitc::RMW_NAND; + case AtomicRMWInst::Or: + return bitc::RMW_OR; + case AtomicRMWInst::Xor: + return bitc::RMW_XOR; + case AtomicRMWInst::Max: + return bitc::RMW_MAX; + case AtomicRMWInst::Min: + return bitc::RMW_MIN; + case AtomicRMWInst::UMax: + return bitc::RMW_UMAX; + case AtomicRMWInst::UMin: + return bitc::RMW_UMIN; + case AtomicRMWInst::FAdd: + return bitc::RMW_FADD; + case AtomicRMWInst::FSub: + return bitc::RMW_FSUB; } } static unsigned getEncodedOrdering(AtomicOrdering Ordering) { switch (Ordering) { - case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC; - case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED; - case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC; - case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE; - case AtomicOrdering::Release: return bitc::ORDERING_RELEASE; - case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL; - case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST; + case AtomicOrdering::NotAtomic: + return bitc::ORDERING_NOTATOMIC; + case AtomicOrdering::Unordered: + return bitc::ORDERING_UNORDERED; + case AtomicOrdering::Monotonic: + return bitc::ORDERING_MONOTONIC; + case AtomicOrdering::Acquire: + return bitc::ORDERING_ACQUIRE; + case AtomicOrdering::Release: + return bitc::ORDERING_RELEASE; + case AtomicOrdering::AcquireRelease: + return bitc::ORDERING_ACQREL; + case AtomicOrdering::SequentiallyConsistent: + return bitc::ORDERING_SEQCST; } llvm_unreachable("Invalid ordering"); } @@ -791,7 +838,8 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() { const std::vector &AttrGrps = VE.getAttributeGroups(); - if (AttrGrps.empty()) return; + if (AttrGrps.empty()) + return; Stream.EnterSubblock(bitc::PARAMATTR_GROUP_BLOCK_ID, 3); @@ -840,7 +888,8 @@ void ModuleBitcodeWriter::writeAttributeTable() { const std::vector &Attrs = VE.getAttributeLists(); - if (Attrs.empty()) return; + if (Attrs.empty()) + return; Stream.EnterSubblock(bitc::PARAMATTR_BLOCK_ID, 3); @@ -872,7 +921,7 @@ auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_POINTER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); - Abbv->Add(BitCodeAbbrevOp(0)); // Addrspace = 0 + Abbv->Add(BitCodeAbbrevOp(0)); // Addrspace = 0 unsigned PtrAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for TYPE_CODE_OPAQUE_POINTER. @@ -884,7 +933,7 @@ // Abbrev for TYPE_CODE_FUNCTION. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); unsigned FunctionAbbrev = Stream.EmitAbbrev(std::move(Abbv)); @@ -892,7 +941,7 @@ // Abbrev for TYPE_CODE_STRUCT_ANON. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); unsigned StructAnonAbbrev = Stream.EmitAbbrev(std::move(Abbv)); @@ -907,7 +956,7 @@ // Abbrev for TYPE_CODE_STRUCT_NAMED. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); unsigned StructNamedAbbrev = Stream.EmitAbbrev(std::move(Abbv)); @@ -915,7 +964,7 @@ // Abbrev for TYPE_CODE_ARRAY. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits)); unsigned ArrayAbbrev = Stream.EmitAbbrev(std::move(Abbv)); @@ -930,19 +979,45 @@ unsigned Code = 0; switch (T->getTypeID()) { - case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break; - case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break; - case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break; - case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break; - case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break; - case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break; - case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break; - case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break; - case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break; - case Type::MetadataTyID: Code = bitc::TYPE_CODE_METADATA; break; - case Type::X86_MMXTyID: Code = bitc::TYPE_CODE_X86_MMX; break; - case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break; - case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break; + case Type::VoidTyID: + Code = bitc::TYPE_CODE_VOID; + break; + case Type::HalfTyID: + Code = bitc::TYPE_CODE_HALF; + break; + case Type::BFloatTyID: + Code = bitc::TYPE_CODE_BFLOAT; + break; + case Type::FloatTyID: + Code = bitc::TYPE_CODE_FLOAT; + break; + case Type::DoubleTyID: + Code = bitc::TYPE_CODE_DOUBLE; + break; + case Type::X86_FP80TyID: + Code = bitc::TYPE_CODE_X86_FP80; + break; + case Type::FP128TyID: + Code = bitc::TYPE_CODE_FP128; + break; + case Type::PPC_FP128TyID: + Code = bitc::TYPE_CODE_PPC_FP128; + break; + case Type::LabelTyID: + Code = bitc::TYPE_CODE_LABEL; + break; + case Type::MetadataTyID: + Code = bitc::TYPE_CODE_METADATA; + break; + case Type::X86_MMXTyID: + Code = bitc::TYPE_CODE_X86_MMX; + break; + case Type::X86_AMXTyID: + Code = bitc::TYPE_CODE_X86_AMX; + break; + case Type::TokenTyID: + Code = bitc::TYPE_CODE_TOKEN; + break; case Type::IntegerTyID: // INTEGER: [width] Code = bitc::TYPE_CODE_INTEGER; @@ -1112,29 +1187,40 @@ static unsigned getEncodedVisibility(const GlobalValue &GV) { switch (GV.getVisibility()) { - case GlobalValue::DefaultVisibility: return 0; - case GlobalValue::HiddenVisibility: return 1; - case GlobalValue::ProtectedVisibility: return 2; + case GlobalValue::DefaultVisibility: + return 0; + case GlobalValue::HiddenVisibility: + return 1; + case GlobalValue::ProtectedVisibility: + return 2; } llvm_unreachable("Invalid visibility"); } static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) { switch (GV.getDLLStorageClass()) { - case GlobalValue::DefaultStorageClass: return 0; - case GlobalValue::DLLImportStorageClass: return 1; - case GlobalValue::DLLExportStorageClass: return 2; + case GlobalValue::DefaultStorageClass: + return 0; + case GlobalValue::DLLImportStorageClass: + return 1; + case GlobalValue::DLLExportStorageClass: + return 2; } llvm_unreachable("Invalid DLL storage class"); } static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) { switch (GV.getThreadLocalMode()) { - case GlobalVariable::NotThreadLocal: return 0; - case GlobalVariable::GeneralDynamicTLSModel: return 1; - case GlobalVariable::LocalDynamicTLSModel: return 2; - case GlobalVariable::InitialExecTLSModel: return 3; - case GlobalVariable::LocalExecTLSModel: return 4; + case GlobalVariable::NotThreadLocal: + return 0; + case GlobalVariable::GeneralDynamicTLSModel: + return 1; + case GlobalVariable::LocalDynamicTLSModel: + return 2; + case GlobalVariable::InitialExecTLSModel: + return 3; + case GlobalVariable::LocalExecTLSModel: + return 4; } llvm_unreachable("Invalid TLS model"); } @@ -1157,9 +1243,12 @@ static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) { switch (GV.getUnnamedAddr()) { - case GlobalValue::UnnamedAddr::None: return 0; - case GlobalValue::UnnamedAddr::Local: return 2; - case GlobalValue::UnnamedAddr::Global: return 1; + case GlobalValue::UnnamedAddr::None: + return 0; + case GlobalValue::UnnamedAddr::Local: + return 2; + case GlobalValue::UnnamedAddr::Global: + return 1; } llvm_unreachable("Invalid unnamed_addr"); } @@ -1229,8 +1318,8 @@ "Sanitizer Metadata is too large for naive serialization."); static unsigned serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) { - return Meta.NoAddress | (Meta.NoHWAddress << 1) | - (Meta.NoMemtag << 2) | (Meta.IsDynInit << 3); + return Meta.NoAddress | (Meta.NoHWAddress << 1) | (Meta.NoMemtag << 2) | + (Meta.IsDynInit << 3); } /// Emit top-level description of module, including target triple, inline asm, @@ -1265,8 +1354,8 @@ // Give section names unique ID's. unsigned &Entry = SectionMap[std::string(GV.getSection())]; if (!Entry) { - writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), - 0 /*TODO*/); + writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, + GV.getSection(), 0 /*TODO*/); Entry = SectionMap.size(); } } @@ -1302,7 +1391,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - Log2_32_Ceil(MaxGlobalType+1))); + Log2_32_Ceil(MaxGlobalType + 1))); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2 //| explicitType << 1 //| constant @@ -1313,13 +1402,13 @@ else { unsigned MaxEncAlignment = getEncodedAlign(MaxAlignment); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - Log2_32_Ceil(MaxEncAlignment+1))); + Log2_32_Ceil(MaxEncAlignment + 1))); } - if (SectionMap.empty()) // Section. + if (SectionMap.empty()) // Section. Abbv->Add(BitCodeAbbrevOp(0)); else Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - Log2_32_Ceil(SectionMap.size()+1))); + Log2_32_Ceil(SectionMap.size() + 1))); // Don't bother emitting vis + thread local. SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv)); } @@ -1361,8 +1450,8 @@ Vals.push_back(GV.getName().size()); Vals.push_back(VE.getTypeID(GV.getValueType())); Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant()); - Vals.push_back(GV.isDeclaration() ? 0 : - (VE.getValueID(GV.getInitializer()) + 1)); + Vals.push_back( + GV.isDeclaration() ? 0 : (VE.getValueID(GV.getInitializer()) + 1)); Vals.push_back(getEncodedLinkage(GV)); Vals.push_back(getEncodedAlign(GV.getAlign())); Vals.push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())] @@ -1418,8 +1507,8 @@ Vals.push_back(getEncodedVisibility(F)); Vals.push_back(F.hasGC() ? GCMap[F.getGC()] : 0); Vals.push_back(getEncodedUnnamedAddr(F)); - Vals.push_back(F.hasPrologueData() ? (VE.getValueID(F.getPrologueData()) + 1) - : 0); + Vals.push_back( + F.hasPrologueData() ? (VE.getValueID(F.getPrologueData()) + 1) : 0); Vals.push_back(getEncodedDLLStorageClass(F)); Vals.push_back(F.hasComdat() ? VE.getComdatID(F.getComdat()) : 0); Vals.push_back(F.hasPrefixData() ? (VE.getValueID(F.getPrefixData()) + 1) @@ -2027,9 +2116,9 @@ Record.clear(); } -void ModuleBitcodeWriter::writeDILabel( - const DILabel *N, SmallVectorImpl &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILabel(const DILabel *N, + SmallVectorImpl &Record, + unsigned Abbrev) { Record.push_back((uint64_t)N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -2180,7 +2269,7 @@ if (MDs.empty()) return; - // Initialize MDNode abbreviations. + // Initialize MDNode abbreviations. #define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0; #include "llvm/IR/Metadata.def" @@ -2345,7 +2434,8 @@ I.getAllMetadataOtherThanDebugLoc(MDs); // If no metadata, ignore instruction. - if (MDs.empty()) continue; + if (MDs.empty()) + continue; Record.push_back(VE.getInstructionID(&I)); @@ -2368,7 +2458,8 @@ SmallVector Names; M.getMDKindNames(Names); - if (Names.empty()) return; + if (Names.empty()) + return; Stream.EnterSubblock(bitc::METADATA_KIND_BLOCK_ID, 3); @@ -2431,7 +2522,8 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal) { - if (FirstVal == LastVal) return; + if (FirstVal == LastVal) + return; Stream.EnterSubblock(bitc::CONSTANTS_BLOCK_ID, 4); @@ -2445,7 +2537,8 @@ auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(LastVal+1))); + Abbv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(LastVal + 1))); AggregateAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for CST_CODE_STRING. @@ -2517,7 +2610,7 @@ emitSignedInt64(Record, V); Code = bitc::CST_CODE_INTEGER; AbbrevToUse = CONSTANTS_INTEGER_ABBREV; - } else { // Wide integers, > 64 bits in size. + } else { // Wide integers, > 64 bits in size. emitWideAPInt(Record, IV->getValue()); Code = bitc::CST_CODE_WIDE_INTEGER; } @@ -2550,7 +2643,7 @@ // If this is a null-terminated string, use the denser CSTRING encoding. if (Str->isCString()) { Code = bitc::CST_CODE_CSTRING; - --NumElts; // Don't encode the null, which isn't allowed by char6. + --NumElts; // Don't encode the null, which isn't allowed by char6. } else { Code = bitc::CST_CODE_STRING; AbbrevToUse = String8Abbrev; @@ -2570,7 +2663,7 @@ else if (isCStr7) AbbrevToUse = CString7Abbrev; } else if (const ConstantDataSequential *CDS = - dyn_cast(C)) { + dyn_cast(C)) { Code = bitc::CST_CODE_DATA; Type *EltTy = CDS->getElementType(); if (isa(EltTy)) { @@ -2878,45 +2971,39 @@ break; } - case Instruction::Ret: - { - Code = bitc::FUNC_CODE_INST_RET; - unsigned NumOperands = I.getNumOperands(); - if (NumOperands == 0) - AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV; - else if (NumOperands == 1) { - if (!pushValueAndType(I.getOperand(0), InstID, Vals)) - AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV; - } else { - for (unsigned i = 0, e = NumOperands; i != e; ++i) - pushValueAndType(I.getOperand(i), InstID, Vals); - } + case Instruction::Ret: { + Code = bitc::FUNC_CODE_INST_RET; + unsigned NumOperands = I.getNumOperands(); + if (NumOperands == 0) + AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV; + else if (NumOperands == 1) { + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) + AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV; + } else { + for (unsigned i = 0, e = NumOperands; i != e; ++i) + pushValueAndType(I.getOperand(i), InstID, Vals); } - break; - case Instruction::Br: - { - Code = bitc::FUNC_CODE_INST_BR; - const BranchInst &II = cast(I); - Vals.push_back(VE.getValueID(II.getSuccessor(0))); - if (II.isConditional()) { - Vals.push_back(VE.getValueID(II.getSuccessor(1))); - pushValue(II.getCondition(), InstID, Vals); - } + } break; + case Instruction::Br: { + Code = bitc::FUNC_CODE_INST_BR; + const BranchInst &II = cast(I); + Vals.push_back(VE.getValueID(II.getSuccessor(0))); + if (II.isConditional()) { + Vals.push_back(VE.getValueID(II.getSuccessor(1))); + pushValue(II.getCondition(), InstID, Vals); } - break; - case Instruction::Switch: - { - Code = bitc::FUNC_CODE_INST_SWITCH; - const SwitchInst &SI = cast(I); - Vals.push_back(VE.getTypeID(SI.getCondition()->getType())); - pushValue(SI.getCondition(), InstID, Vals); - Vals.push_back(VE.getValueID(SI.getDefaultDest())); - for (auto Case : SI.cases()) { - Vals.push_back(VE.getValueID(Case.getCaseValue())); - Vals.push_back(VE.getValueID(Case.getCaseSuccessor())); - } + } break; + case Instruction::Switch: { + Code = bitc::FUNC_CODE_INST_SWITCH; + const SwitchInst &SI = cast(I); + Vals.push_back(VE.getTypeID(SI.getCondition()->getType())); + pushValue(SI.getCondition(), InstID, Vals); + Vals.push_back(VE.getValueID(SI.getDefaultDest())); + for (auto Case : SI.cases()) { + Vals.push_back(VE.getValueID(Case.getCaseValue())); + Vals.push_back(VE.getValueID(Case.getCaseSuccessor())); } - break; + } break; case Instruction::IndirectBr: Code = bitc::FUNC_CODE_INST_INDIRECTBR; Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); @@ -3209,9 +3296,9 @@ } case Instruction::VAArg: Code = bitc::FUNC_CODE_INST_VAARG; - Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); // valistty - pushValue(I.getOperand(0), InstID, Vals); // valist. - Vals.push_back(VE.getTypeID(I.getType())); // restype. + Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); // valistty + pushValue(I.getOperand(0), InstID, Vals); // valist. + Vals.push_back(VE.getTypeID(I.getType())); // restype. break; case Instruction::Freeze: Code = bitc::FUNC_CODE_INST_FREEZE; @@ -3226,7 +3313,7 @@ /// Write a GlobalValue VST to the module. The purpose of this data structure is /// to allow clients to efficiently find the function body. void ModuleBitcodeWriter::writeGlobalValueSymbolTable( - DenseMap &FunctionToBitcodeIndex) { + DenseMap &FunctionToBitcodeIndex) { // Get the offset of the VST we are writing, and backpatch it into // the VST forward declaration record. uint64_t VSTOffset = Stream.GetCurrentBitNo(); @@ -3523,10 +3610,10 @@ { // CE_CAST abbrev for CONSTANTS_BLOCK. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid VE.computeBitsRequiredForTypeIndicies())); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != CONSTANTS_CE_CAST_Abbrev) @@ -3548,7 +3635,7 @@ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty VE.computeBitsRequiredForTypeIndicies())); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_LOAD_ABBREV) @@ -3557,7 +3644,7 @@ { // INST_UNOP abbrev for FUNCTION_BLOCK. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_UNOP_ABBREV) @@ -3566,7 +3653,7 @@ { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != @@ -3576,8 +3663,8 @@ { // INST_BINOP abbrev for FUNCTION_BLOCK. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_BINOP_ABBREV) @@ -3586,8 +3673,8 @@ { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != @@ -3597,10 +3684,10 @@ { // INST_CAST abbrev for FUNCTION_BLOCK. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty VE.computeBitsRequiredForTypeIndicies())); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != FUNCTION_INST_CAST_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); @@ -4006,13 +4093,13 @@ // Abbrev for FS_PERMODULE_PROFILE. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -4024,13 +4111,13 @@ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF)); else Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid [, rel_block_freq]) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -4059,9 +4146,9 @@ // Abbrev for FS_ALIAS. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_TYPE_ID_METADATA @@ -4149,15 +4236,15 @@ // Abbrev for FS_COMBINED. auto Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -4166,15 +4253,15 @@ // Abbrev for FS_COMBINED_PROFILE. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -4183,20 +4270,20 @@ // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // Abbrev for FS_COMBINED_ALIAS. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); // The aliases are emitted as a post-pass, and will point to the value @@ -4537,10 +4624,10 @@ // number from /usr/include/mach/machine.h. It is ok to reproduce the // specific constants here because they are implicitly part of the Darwin ABI. enum { - DARWIN_CPU_ARCH_ABI64 = 0x01000000, - DARWIN_CPU_TYPE_X86 = 7, - DARWIN_CPU_TYPE_ARM = 12, - DARWIN_CPU_TYPE_POWERPC = 18 + DARWIN_CPU_ARCH_ABI64 = 0x01000000, + DARWIN_CPU_TYPE_X86 = 7, + DARWIN_CPU_TYPE_ARM = 12, + DARWIN_CPU_TYPE_POWERPC = 18 }; Triple::ArchType Arch = TT.getArch(); @@ -4689,7 +4776,7 @@ const ModuleSummaryIndex *Index, bool GenerateHash, ModuleHash *ModHash) { SmallVector Buffer; - Buffer.reserve(256*1024); + Buffer.reserve(256 * 1024); // If this is darwin or another generic macho target, reserve space for the // header. @@ -4928,6 +5015,37 @@ llvm_unreachable("Unimplemented ObjectFormatType"); } +static const char *getSectionNameForBitcodeForFatLTO(const Triple &T) { + switch (T.getObjectFormat()) { + case Triple::MachO: + llvm_unreachable("MachO is not yet implemented"); + break; + case Triple::COFF: + llvm_unreachable("COFF is not yet implemented"); + break; + case Triple::ELF: + return ".fatlto"; + case Triple::Wasm: + llvm_unreachable("Wasm is not yet implemented"); + break; + case Triple::UnknownObjectFormat: + return ".fatlto"; + case Triple::GOFF: + llvm_unreachable("GOFF is not yet implemented"); + break; + case Triple::SPIRV: + llvm_unreachable("SPIRV is not yet implemented"); + break; + case Triple::XCOFF: + llvm_unreachable("XCOFF is not yet implemented"); + break; + case Triple::DXContainer: + llvm_unreachable("DXContainer is not yet implemented"); + break; + } + llvm_unreachable("Unimplemented ObjectFormatType"); +} + static const char *getSectionNameForCommandline(const Triple &T) { switch (T.getObjectFormat()) { case Triple::MachO: @@ -5045,3 +5163,68 @@ llvm::ConstantArray::get(ATy, UsedArray), "llvm.compiler.used"); NewUsed->setSection("llvm.metadata"); } + +void llvm::embedBitcodeInFatObject(llvm::Module &M, llvm::MemoryBufferRef Buf) { + // Save llvm.compiler.used and remove it. + SmallVector UsedArray; + SmallVector UsedGlobals; + Type *UsedElementType = Type::getInt8Ty(M.getContext())->getPointerTo(0); + GlobalVariable *Used = collectUsedGlobalVariables(M, UsedGlobals, true); + for (auto *GV : UsedGlobals) { + if (GV->getName() != "llvm.embedded.module" && + GV->getName() != "llvm.cmdline") + UsedArray.push_back( + ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType)); + } + if (Used) + Used->eraseFromParent(); + + // Embed the bitcode for the llvm module. + std::string Data; + ArrayRef ModuleData; + Triple T(M.getTargetTriple()); + + if (Buf.getBufferSize() == 0 || + !isBitcode((const unsigned char *)Buf.getBufferStart(), + (const unsigned char *)Buf.getBufferEnd())) { + // If the input is LLVM Assembly, bitcode is produced by serializing + // the module. Use-lists order need to be preserved in this case. + llvm::raw_string_ostream OS(Data); + llvm::WriteBitcodeToFile(M, OS, /* ShouldPreserveUseListOrder */ true); + ModuleData = + ArrayRef((const uint8_t *)OS.str().data(), OS.str().size()); + } else + // If the input is LLVM bitcode, write the input byte stream directly. + ModuleData = ArrayRef((const uint8_t *)Buf.getBufferStart(), + Buf.getBufferSize()); + llvm::Constant *ModuleConstant = + llvm::ConstantDataArray::get(M.getContext(), ModuleData); + llvm::GlobalVariable *GV = new llvm::GlobalVariable( + M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage, + ModuleConstant); + GV->setSection(getSectionNameForBitcodeForFatLTO(T)); + // Set alignment to 1 to prevent padding between two contributions from input + // sections after linking. + GV->setAlignment(Align(1)); + UsedArray.push_back( + ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType)); + if (llvm::GlobalVariable *Old = + M.getGlobalVariable("llvm.embedded.module", true)) { + assert(Old->hasZeroLiveUses() && + "llvm.embedded.module can only be used once in llvm.compiler.used"); + GV->takeName(Old); + Old->eraseFromParent(); + } else { + GV->setName("llvm.embedded.module"); + } + + if (UsedArray.empty()) + return; + + // Recreate llvm.compiler.used. + ArrayType *ATy = ArrayType::get(UsedElementType, UsedArray.size()); + auto *NewUsed = new GlobalVariable( + M, ATy, false, llvm::GlobalValue::AppendingLinkage, + llvm::ConstantArray::get(ATy, UsedArray), "llvm.compiler.used"); + NewUsed->setSection("llvm.metadata"); +}