Index: llvm/include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- llvm/include/llvm/IR/ModuleSummaryIndex.h +++ llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -941,6 +941,11 @@ /// considered live. bool WithGlobalValueDeadStripping = false; + /// Indicates that summary-based attribute propagation has run and + /// GVarFlags::MaybeReadonly / GVarFlags::MaybeWriteonly are really + /// read/write only. + bool WithAttributePropagation = false; + /// Indicates that summary-based synthetic entry count propagation has run bool HasSyntheticEntryCounts = false; @@ -1065,6 +1070,18 @@ WithGlobalValueDeadStripping = true; } + bool withAttributePropagation() const { return WithAttributePropagation; } + void setWithAttributePropagation() { + WithAttributePropagation = true; + } + + bool isReadOnly(const GlobalVarSummary *GVS) const { + return WithAttributePropagation && GVS->maybeReadOnly(); + } + bool isWriteOnly(const GlobalVarSummary *GVS) const { + return WithAttributePropagation && GVS->maybeWriteOnly(); + } + bool hasSyntheticEntryCounts() const { return HasSyntheticEntryCounts; } void setHasSyntheticEntryCounts() { HasSyntheticEntryCounts = true; } @@ -1356,6 +1373,9 @@ /// Analyze index and detect unmodified globals void propagateAttributes(const DenseSet &PreservedSymbols); + + /// Checks if we can import global variable from another module. + bool canImportGlobalVar(GlobalValueSummary *S, bool AnalyzeRefs) const; }; /// GraphTraits definition to build SCC for the index @@ -1427,15 +1447,6 @@ return ValueInfo(I->haveGVs(), &P); } }; - -static inline bool canImportGlobalVar(GlobalValueSummary *S) { - assert(isa(S->getBaseObject())); - - // We don't import GV with references, because it can result - // in promotion of local variables in the source module. - return !GlobalValue::isInterposableLinkage(S->linkage()) && - !S->notEligibleToImport() && S->refs().empty(); -} } // end namespace llvm #endif // LLVM_IR_MODULESUMMARYINDEX_H Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5763,7 +5763,7 @@ } const uint64_t Version = Record[0]; const bool IsOldProfileFormat = Version == 1; - if (Version < 1 || Version > 7) + if (Version < 1 || Version > 8) return error("Invalid summary version " + Twine(Version) + ". Version should be in the range [1-7]."); Record.clear(); @@ -5816,7 +5816,7 @@ case bitc::FS_FLAGS: { // [flags] uint64_t Flags = Record[0]; // Scan flags. - assert(Flags <= 0x1f && "Unexpected bits in flag"); + assert(Flags <= 0x3f && "Unexpected bits in flag"); // 1 bit: WithGlobalValueDeadStripping flag. // Set on combined index only. @@ -5839,6 +5839,10 @@ // Set on combined index only. if (Flags & 0x10) TheIndex.setPartiallySplitLTOUnits(); + // 1 bit: WithAttributePropagation flag. + // Set on combined index only. + if (Flags & 0x20) + TheIndex.setWithAttributePropagation(); break; } case bitc::FS_VALUE_GUID: { // [valueid, refguid] @@ -6554,7 +6558,7 @@ case bitc::FS_FLAGS: { // [flags] uint64_t Flags = Record[0]; // Scan flags. - assert(Flags <= 0x1f && "Unexpected bits in flag"); + assert(Flags <= 0x3f && "Unexpected bits in flag"); return Flags & 0x8; } Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3723,7 +3723,7 @@ // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 7; +static const uint64_t INDEX_VERSION = 8; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. @@ -3899,6 +3899,8 @@ Flags |= 0x8; if (Index.partiallySplitLTOUnits()) Flags |= 0x10; + if (Index.withAttributePropagation()) + Flags |= 0x20; Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef{Flags}); for (const auto &GVI : valueIds()) { Index: llvm/lib/IR/ModuleSummaryIndex.cpp =================================================================== --- llvm/lib/IR/ModuleSummaryIndex.cpp +++ llvm/lib/IR/ModuleSummaryIndex.cpp @@ -173,7 +173,7 @@ if (auto *GVS = dyn_cast(S->getBaseObject())) // Here we intentionally pass S.get() not GVS, because S could be // an alias. - if (!canImportGlobalVar(S.get()) || + if (!canImportGlobalVar(S.get(), false) || GUIDPreservedSymbols.count(P.first)) { GVS->setReadOnly(false); GVS->setWriteOnly(false); @@ -193,6 +193,23 @@ } } +bool ModuleSummaryIndex::canImportGlobalVar(GlobalValueSummary *S, + bool AnalyzeRefs) const { + auto HasRefsPreventingImport = [this](const GlobalVarSummary *GVS) { + return !isReadOnly(GVS) && GVS->refs().size(); + }; + auto *GVS = cast(S->getBaseObject()); + + // Global variable with non-trivial initializer can be imported + // if it's readonly. This gives us extra opportunities for constant + // folding and converting indirect calls to direct calls. We don't + // analyze GV references during attribute propagation, because we + // don't know yet if it is readonly or not. + return !GlobalValue::isInterposableLinkage(S->linkage()) && + !S->notEligibleToImport() && + (!AnalyzeRefs || !HasRefsPreventingImport(GVS)); +} + // TODO: write a graphviz dumper for SCCs (see ModuleSummaryIndex::exportToDot) // then delete this function and update its tests LLVM_DUMP_METHOD Index: llvm/lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- llvm/lib/Transforms/IPO/FunctionImport.cpp +++ llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -280,7 +280,8 @@ } static void computeImportForReferencedGlobals( - const FunctionSummary &Summary, const GVSummaryMapTy &DefinedGVSummaries, + const FunctionSummary &Summary, const ModuleSummaryIndex &Index, + const GVSummaryMapTy &DefinedGVSummaries, FunctionImporter::ImportMapTy &ImportList, StringMap *ExportLists) { for (auto &VI : Summary.refs()) { @@ -303,16 +304,24 @@ RefSummary->modulePath() != Summary.modulePath(); }; + auto MarkExported = [&](const ValueInfo &VI, const GlobalValueSummary *S) { + if (ExportLists) + (*ExportLists)[S->modulePath()].insert(VI.getGUID()); + }; + for (auto &RefSummary : VI.getSummaryList()) if (isa(RefSummary.get()) && - canImportGlobalVar(RefSummary.get()) && + Index.canImportGlobalVar(RefSummary.get(), true) && !LocalNotInModule(RefSummary.get())) { auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID()); // Only update stat if we haven't already imported this variable. if (ILI.second) NumImportedGlobalVarsThinLink++; - if (ExportLists) - (*ExportLists)[RefSummary->modulePath()].insert(VI.getGUID()); + MarkExported(VI, RefSummary.get()); + // Promote referenced functions and variables + for (const auto &VI : RefSummary->refs()) + for (const auto &RefFn : VI.getSummaryList()) + MarkExported(VI, RefFn.get()); break; } } @@ -351,8 +360,8 @@ FunctionImporter::ImportMapTy &ImportList, StringMap *ExportLists, FunctionImporter::ImportThresholdsTy &ImportThresholds) { - computeImportForReferencedGlobals(Summary, DefinedGVSummaries, ImportList, - ExportLists); + computeImportForReferencedGlobals(Summary, Index, DefinedGVSummaries, + ImportList, ExportLists); static int ImportCount = 0; for (auto &Edge : Summary.calls()) { ValueInfo VI = Edge.first; @@ -852,6 +861,10 @@ function_ref isPrevailing, bool ImportEnabled) { computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing); + // This function can be run twice: once with empty list of preserved + // symbols and second time when some preserved symbols are given. + // We still need to decide if we have to import global variable correctly + // on such occasion. See ModuleSummaryIndex::canImportGlobalVar. if (ImportEnabled) { Index.propagateAttributes(GUIDPreservedSymbols); } else { @@ -864,6 +877,7 @@ GVS->setWriteOnly(false); } } + Index.setWithAttributePropagation(); } /// Compute the set of summaries needed for a ThinLTO backend compilation of Index: llvm/lib/Transforms/Utils/FunctionImportUtils.cpp =================================================================== --- llvm/lib/Transforms/Utils/FunctionImportUtils.cpp +++ llvm/lib/Transforms/Utils/FunctionImportUtils.cpp @@ -238,7 +238,7 @@ // If global value dead stripping is not enabled in summary then // propagateConstants hasn't been run. We can't internalize GV // in such case. - if (!GV.isDeclaration() && VI && ImportIndex.withGlobalValueDeadStripping()) { + if (!GV.isDeclaration() && VI && ImportIndex.withAttributePropagation()) { if (GlobalVariable *V = dyn_cast(&GV)) { // We can have more than one local with the same GUID, in the case of // same-named locals in different but same-named source files that were @@ -247,7 +247,7 @@ auto* GVS = dyn_cast( ImportIndex.findSummaryInModule(VI, M.getModuleIdentifier())); // At this stage "maybe" is "definitely" - if (GVS && (GVS->maybeReadOnly() || GVS->maybeWriteOnly())) + if (GVS && (ImportIndex.isReadOnly(GVS) || ImportIndex.isWriteOnly(GVS))) V->addAttribute("thinlto-internalize"); } } Index: llvm/test/Bitcode/summary_version.ll =================================================================== --- llvm/test/Bitcode/summary_version.ll +++ llvm/test/Bitcode/summary_version.ll @@ -2,7 +2,7 @@ ; RUN: opt -module-summary %s -o - | llvm-bcanalyzer -dump | FileCheck %s ; CHECK: +; CHECK: Index: llvm/test/Bitcode/thinlto-deadstrip-flag.ll =================================================================== --- llvm/test/Bitcode/thinlto-deadstrip-flag.ll +++ llvm/test/Bitcode/thinlto-deadstrip-flag.ll @@ -5,14 +5,14 @@ ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \ ; RUN: -r %t.o,glob,plx ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=WITHDEAD -; WITHDEAD: +; WITHDEAD: ; Ensure dead stripping performed flag is not set on distributed index ; when option used to disable dead stripping computation. ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \ ; RUN: -r %t.o,glob,plx -compute-dead=false ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=NODEAD -; NODEAD: +; NODEAD: target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/test/Bitcode/thinlto-synthetic-count-flag.ll =================================================================== --- llvm/test/Bitcode/thinlto-synthetic-count-flag.ll +++ llvm/test/Bitcode/thinlto-synthetic-count-flag.ll @@ -5,7 +5,7 @@ ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \ ; RUN: -r %t.o,glob,plx -compute-dead=false ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=NOSYNTHETIC -; NOSYNTHETIC: +; NOSYNTHETIC: ; Ensure synthetic entry count flag is set on distributed index ; when option used to enable synthetic count propagation @@ -13,7 +13,7 @@ ; RUN: -r %t.o,glob,plx -thinlto-synthesize-entry-counts \ ; RUN: -compute-dead=false ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=HASSYNTHETIC -; HASSYNTHETIC: +; HASSYNTHETIC: target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" Index: llvm/test/ThinLTO/X86/globals-import.ll =================================================================== --- llvm/test/ThinLTO/X86/globals-import.ll +++ llvm/test/ThinLTO/X86/globals-import.ll @@ -15,7 +15,7 @@ ; RUN: llvm-dis %t2.bc.thinlto.promoted.bc -o - | FileCheck --check-prefix=PROMOTE1 %s ; RUN: llvm-dis %t2b.bc.thinlto.promoted.bc -o - | FileCheck --check-prefix=PROMOTE2 %s -; IMPORT: @baz.llvm.0 = available_externally hidden constant i32 10, align 4 +; IMPORT: @baz.llvm.0 = internal constant i32 10, align 4 ; PROMOTE1: @baz.llvm.0 = hidden constant i32 10, align 4 ; PROMOTE1: define weak_odr i32 @foo() { Index: llvm/test/ThinLTO/X86/local_name_conflict.ll =================================================================== --- llvm/test/ThinLTO/X86/local_name_conflict.ll +++ llvm/test/ThinLTO/X86/local_name_conflict.ll @@ -12,7 +12,7 @@ ; that module (%t3.bc) to be imported. Check that the imported reference's ; promoted name matches the imported copy. ; RUN: llvm-lto -thinlto-action=import %t.bc -thinlto-index=%t4.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=IMPORT -; IMPORT: @baz.llvm.[[HASH:[0-9]+]] = available_externally hidden constant i32 10, align 4 +; IMPORT: @baz.llvm.[[HASH:[0-9]+]] = internal constant i32 10, align 4 ; IMPORT: call i32 @foo.llvm.[[HASH]] ; IMPORT: define available_externally hidden i32 @foo.llvm.[[HASH]]() Index: llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll =================================================================== --- llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll +++ llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll @@ -35,7 +35,7 @@ ; should not be set. ; RUN: llvm-bcanalyzer --dump %t1.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC1 ; CHECK-BC1: +; CHECK-BC1: ; CHECK-BC1: