diff --git a/llvm/include/llvm/Bitstream/BitstreamReader.h b/llvm/include/llvm/Bitstream/BitstreamReader.h --- a/llvm/include/llvm/Bitstream/BitstreamReader.h +++ b/llvm/include/llvm/Bitstream/BitstreamReader.h @@ -193,7 +193,7 @@ assert(NumBits && NumBits <= BitsInWord && "Cannot return zero or more than BitsInWord bits!"); - static const unsigned Mask = sizeof(word_t) > 4 ? 0x3f : 0x1f; + static const unsigned Mask = sizeof(word_t) > 4 ? 0x7f : 0x1f; // If the field is fully contained by CurWord, return it quickly. if (BitsInCurWord >= NumBits) { diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -228,7 +228,9 @@ /// protected and hidden. GlobalValue::VisibilityTypes getELFVisibility() const; - bool isDSOLocal() const; + /// Checks if all summaries are DSO local (have the flag set). When DSOLocal + /// propagation has been done, set the parameter to enable fast check. + bool isDSOLocal(bool WithDSOLocalPropagation = false) const; /// Checks if all copies are eligible for auto-hiding (have flag set). bool canAutoHide() const; @@ -1055,6 +1057,10 @@ /// read/write only. bool WithAttributePropagation = false; + /// Indicates that summary-based DSOLocal propagation has run and the flag in + /// every summary of a GV is synchronized. + bool WithDSOLocalPropagation = false; + /// Indicates that summary-based synthetic entry count propagation has run bool HasSyntheticEntryCounts = false; @@ -1210,6 +1216,9 @@ WithAttributePropagation = true; } + bool withDSOLocalPropagation() const { return WithDSOLocalPropagation; } + void setWithDSOLocalPropagation() { WithDSOLocalPropagation = true; } + bool isReadOnly(const GlobalVarSummary *GVS) const { return WithAttributePropagation && GVS->maybeReadOnly(); } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -6806,7 +6806,7 @@ case bitc::FS_FLAGS: { // [flags] uint64_t Flags = Record[0]; // Scan flags. - assert(Flags <= 0x3f && "Unexpected bits in flag"); + assert(Flags <= 0x7f && "Unexpected bits in flag"); return Flags & 0x8; } diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp --- a/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -52,13 +52,17 @@ : GlobalValue::DefaultVisibility; } -bool ValueInfo::isDSOLocal() const { - // Need to check all summaries are local in case of hash collisions. - return getSummaryList().size() && - llvm::all_of(getSummaryList(), - [](const std::unique_ptr &Summary) { - return Summary->isDSOLocal(); - }); +bool ValueInfo::isDSOLocal(bool WithDSOLocalPropagation) const { + // With DSOLocal propagation done, the flag in evey summary is the same. + // Check the first one is enough. + return WithDSOLocalPropagation + ? getSummaryList().size() && getSummaryList()[0]->isDSOLocal() + : getSummaryList().size() && + llvm::all_of( + getSummaryList(), + [](const std::unique_ptr &Summary) { + return Summary->isDSOLocal(); + }); } bool ValueInfo::canAutoHide() const { @@ -100,11 +104,13 @@ Flags |= 0x10; if (withAttributePropagation()) Flags |= 0x20; + if (withDSOLocalPropagation()) + Flags |= 0x40; return Flags; } void ModuleSummaryIndex::setFlags(uint64_t Flags) { - assert(Flags <= 0x3f && "Unexpected bits in flag"); + assert(Flags <= 0x7f && "Unexpected bits in flag"); // 1 bit: WithGlobalValueDeadStripping flag. // Set on combined index only. if (Flags & 0x1) @@ -130,6 +136,10 @@ // Set on combined index only. if (Flags & 0x20) setWithAttributePropagation(); + // 1 bit: WithDSOLocalPropagation flag. + // Set on combined index only. + if (Flags & 0x40) + setWithDSOLocalPropagation(); } // Collect for the given module the list of function it defines @@ -205,7 +215,7 @@ } } -// Do the access attribute propagation in combined index. +// Do the access attribute and DSOLocal propagation in combined index. // The goal of attribute propagation is internalization of readonly (RO) // or writeonly (WO) variables. To determine which variables are RO or WO // and which are not we take following steps: @@ -216,7 +226,7 @@ // or doesn't read it (writeonly). // // - After computing dead symbols in combined index we do the attribute -// propagation. During this step we: +// and DSOLocal propagation. During this step we: // a. clear RO and WO attributes from variables which are preserved or // can't be imported // b. clear RO and WO attributes from variables referenced by any global @@ -225,6 +235,7 @@ // reference is not readonly // d. clear WO attribute from variable referenced by a function when // reference is not writeonly +// e. clear IsDSOLocal flag in every summary if any of them is false. // // Because of (c, d) we don't internalize variables read by function A // and modified by function B. @@ -236,7 +247,8 @@ if (!PropagateAttrs) return; DenseSet MarkedNonReadWriteOnly; - for (auto &P : *this) + for (auto &P : *this) { + bool IsDSOLocal = true; for (auto &S : P.second.SummaryList) { if (!isGlobalValueLive(S.get())) { // computeDeadSymbols should have marked all copies live. Note that @@ -273,8 +285,20 @@ GVS->setWriteOnly(false); } propagateAttributesToRefs(S.get(), MarkedNonReadWriteOnly); + + // If the flag from any summary is false, the GV is not DSOLocal. + IsDSOLocal &= S->isDSOLocal(); } + if (!IsDSOLocal) + // Mark the flag in all summaries false so that we can do quick check + // without going through the whole list. + llvm::for_each(P.second.SummaryList, + [](const std::unique_ptr &Summary) { + return Summary->setDSOLocal(false); + }); + } setWithAttributePropagation(); + setWithDSOLocalPropagation(); if (llvm::AreStatisticsEnabled()) for (auto &P : *this) if (P.second.SummaryList.size()) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -207,7 +207,7 @@ AddUnsigned(GS->isLive()); AddUnsigned(GS->canAutoHide()); for (const ValueInfo &VI : GS->refs()) { - AddUnsigned(VI.isDSOLocal()); + AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation())); AddUsedCfiGlobal(VI.getGUID()); } if (auto *GVS = dyn_cast(GS)) { @@ -226,7 +226,7 @@ for (auto &TT : FS->type_checked_load_const_vcalls()) UsedTypeIds.insert(TT.VFunc.GUID); for (auto &ET : FS->calls()) { - AddUnsigned(ET.first.isDSOLocal()); + AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation())); AddUsedCfiGlobal(ET.first.getGUID()); } } diff --git a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp --- a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp +++ b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp @@ -279,7 +279,7 @@ if (ClearDSOLocalOnDeclarations && GV.isDeclarationForLinker() && !GV.isImplicitDSOLocal()) { GV.setDSOLocal(false); - } else if (VI && VI.isDSOLocal()) { + } else if (VI && VI.isDSOLocal(ImportIndex.withDSOLocalPropagation())) { // If all summaries are dso_local, symbol gets resolved to a known local // definition. GV.setDSOLocal(true); diff --git a/llvm/test/Assembler/summary-flags.ll b/llvm/test/Assembler/summary-flags.ll --- a/llvm/test/Assembler/summary-flags.ll +++ b/llvm/test/Assembler/summary-flags.ll @@ -5,8 +5,8 @@ ; RUN: llvm-as %s -o - | llvm-dis -o - | FileCheck %s ; CHECK: ^0 = module ; CHECK-NEXT: ^1 = gv -; CHECK-NEXT: ^2 = flags: 33 +; CHECK-NEXT: ^2 = flags: 97 ^0 = module: (path: "main.bc", hash: (3499594384, 1671013073, 3271036935, 1830411232, 59290952)) ^1 = gv: (guid: 15822663052811949562, summaries: (function: (module: ^0, flags: (linkage: external, notEligibleToImport: 0, live: 1, dsoLocal: 1, canAutoHide: 0), insts: 2))) -^2 = flags: 33 +^2 = flags: 97 diff --git a/llvm/test/Bitcode/thinlto-deadstrip-flag.ll b/llvm/test/Bitcode/thinlto-deadstrip-flag.ll --- a/llvm/test/Bitcode/thinlto-deadstrip-flag.ll +++ b/llvm/test/Bitcode/thinlto-deadstrip-flag.ll @@ -5,14 +5,14 @@ ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \ ; RUN: -r %t.o,glob,plx ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=WITHDEAD -; WITHDEAD: +; WITHDEAD: ; Ensure dead stripping performed flag is not set on distributed index ; when option used to disable dead stripping computation. ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \ ; RUN: -r %t.o,glob,plx -compute-dead=false ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=NODEAD -; NODEAD: +; NODEAD: target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll b/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll --- a/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll +++ b/llvm/test/Bitcode/thinlto-synthetic-count-flag.ll @@ -5,7 +5,7 @@ ; RUN: llvm-lto2 run %t.o -o %t.out -thinlto-distributed-indexes \ ; RUN: -r %t.o,glob,plx -compute-dead=false ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=NOSYNTHETIC -; NOSYNTHETIC: +; NOSYNTHETIC: ; Ensure synthetic entry count flag is set on distributed index ; when option used to enable synthetic count propagation @@ -13,7 +13,7 @@ ; RUN: -r %t.o,glob,plx -thinlto-synthesize-entry-counts \ ; RUN: -compute-dead=false ; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=HASSYNTHETIC -; HASSYNTHETIC: +; HASSYNTHETIC: target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"