Index: llvm/trunk/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ llvm/trunk/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/IPO.h" +#include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/TypeMetadataUtils.h" #include "llvm/Bitcode/BitcodeWriter.h" @@ -25,6 +26,7 @@ #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/ScopedPrinter.h" +#include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/Utils/Cloning.h" using namespace llvm; @@ -236,10 +238,19 @@ } } +void forEachVirtualFunction(Constant *C, function_ref Fn) { + if (auto *F = dyn_cast(C)) + return Fn(F); + for (Value *Op : C->operands()) + forEachVirtualFunction(cast(Op), Fn); +} + // If it's possible to split M into regular and thin LTO parts, do so and write // a multi-module bitcode file with the two parts to OS. Otherwise, write only a // regular LTO bitcode file to OS. -void splitAndWriteThinLTOBitcode(raw_ostream &OS, Module &M) { +void splitAndWriteThinLTOBitcode( + raw_ostream &OS, function_ref AARGetter, + Module &M) { std::string ModuleId = getModuleId(&M); if (ModuleId.empty()) { // We couldn't generate a module ID for this module, just write it out as a @@ -250,21 +261,71 @@ promoteTypeIds(M, ModuleId); - auto IsInMergedM = [&](const GlobalValue *GV) { - auto *GVar = dyn_cast(GV->getBaseObject()); - if (!GVar) - return false; - + // Returns whether a global has attached type metadata. Such globals may + // participate in CFI or whole-program devirtualization, so they need to + // appear in the merged module instead of the thin LTO module. + auto HasTypeMetadata = [&](const GlobalObject *GO) { SmallVector MDs; - GVar->getMetadata(LLVMContext::MD_type, MDs); + GO->getMetadata(LLVMContext::MD_type, MDs); return !MDs.empty(); }; + // Collect the set of virtual functions that are eligible for virtual constant + // propagation. Each eligible function must not access memory, must return + // an integer of width <=64 bits, must take at least one argument, must not + // use its first argument (assumed to be "this") and all arguments other than + // the first one must be of <=64 bit integer type. + // + // Note that we test whether this copy of the function is readnone, rather + // than testing function attributes, which must hold for any copy of the + // function, even a less optimized version substituted at link time. This is + // sound because the virtual constant propagation optimizations effectively + // inline all implementations of the virtual function into each call site, + // rather than using function attributes to perform local optimization. + std::set EligibleVirtualFns; + for (GlobalVariable &GV : M.globals()) + if (HasTypeMetadata(&GV)) + forEachVirtualFunction(GV.getInitializer(), [&](Function *F) { + auto *RT = dyn_cast(F->getReturnType()); + if (!RT || RT->getBitWidth() > 64 || F->arg_empty() || + !F->arg_begin()->use_empty()) + return; + for (auto &Arg : make_range(std::next(F->arg_begin()), F->arg_end())) { + auto *ArgT = dyn_cast(Arg.getType()); + if (!ArgT || ArgT->getBitWidth() > 64) + return; + } + if (computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) == MAK_ReadNone) + EligibleVirtualFns.insert(F); + }); + ValueToValueMapTy VMap; - std::unique_ptr MergedM(CloneModule(&M, VMap, IsInMergedM)); + std::unique_ptr MergedM( + CloneModule(&M, VMap, [&](const GlobalValue *GV) -> bool { + if (auto *F = dyn_cast(GV)) + return EligibleVirtualFns.count(F); + if (auto *GVar = dyn_cast_or_null(GV->getBaseObject())) + return HasTypeMetadata(GVar); + return false; + })); StripDebugInfo(*MergedM); - filterModule(&M, [&](const GlobalValue *GV) { return !IsInMergedM(GV); }); + for (Function &F : *MergedM) + if (!F.isDeclaration()) { + // Reset the linkage of all functions eligible for virtual constant + // propagation. The canonical definitions live in the thin LTO module so + // that they can be imported. + F.setLinkage(GlobalValue::AvailableExternallyLinkage); + F.setComdat(nullptr); + } + + // Remove all globals with type metadata, as well as aliases pointing to them, + // from the thin LTO module. + filterModule(&M, [&](const GlobalValue *GV) { + if (auto *GVar = dyn_cast_or_null(GV->getBaseObject())) + return !HasTypeMetadata(GVar); + return true; + }); promoteInternals(*MergedM, M, ModuleId); promoteInternals(M, *MergedM, ModuleId); @@ -296,11 +357,12 @@ return false; } -void writeThinLTOBitcode(raw_ostream &OS, Module &M, - const ModuleSummaryIndex *Index) { +void writeThinLTOBitcode(raw_ostream &OS, + function_ref AARGetter, + Module &M, const ModuleSummaryIndex *Index) { // See if this module has any type metadata. If so, we need to split it. if (requiresSplit(M)) - return splitAndWriteThinLTOBitcode(OS, M); + return splitAndWriteThinLTOBitcode(OS, AARGetter, M); // Otherwise we can just write it out as a regular module. WriteBitcodeToFile(&M, OS, /*ShouldPreserveUseListOrder=*/false, Index, @@ -326,12 +388,14 @@ bool runOnModule(Module &M) override { const ModuleSummaryIndex *Index = &(getAnalysis().getIndex()); - writeThinLTOBitcode(OS, M, Index); + writeThinLTOBitcode(OS, LegacyAARGetter(*this), M, Index); return true; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); + AU.addRequired(); AU.addRequired(); + AU.addRequired(); } }; } // anonymous namespace @@ -339,7 +403,9 @@ char WriteThinLTOBitcode::ID = 0; INITIALIZE_PASS_BEGIN(WriteThinLTOBitcode, "write-thinlto-bitcode", "Write ThinLTO Bitcode", false, true) +INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode", "Write ThinLTO Bitcode", false, true) Index: llvm/trunk/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll =================================================================== --- llvm/trunk/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll +++ llvm/trunk/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll @@ -0,0 +1,21 @@ +; RUN: opt -thinlto-bc -o %t %s +; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s +; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s + +define [1 x i8*]* @source() { + ret [1 x i8*]* @g +} + +; M0: @"g$84f59439b469192440047efc8de357fb" = external hidden constant [1 x i8*]{{$}} +; M1: @"g$84f59439b469192440047efc8de357fb" = hidden constant [1 x i8*] [i8* bitcast (i64 (i8*)* @"ok$84f59439b469192440047efc8de357fb" to i8*)] +@g = internal constant [1 x i8*] [ + i8* bitcast (i64 (i8*)* @ok to i8*) +], !type !0 + +; M0: define hidden i64 @"ok$84f59439b469192440047efc8de357fb" +; M1: define available_externally hidden i64 @"ok$84f59439b469192440047efc8de357fb" +define internal i64 @ok(i8* %this) { + ret i64 42 +} + +!0 = !{i32 0, !"typeid"} Index: llvm/trunk/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll =================================================================== --- llvm/trunk/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll +++ llvm/trunk/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll @@ -0,0 +1,75 @@ +; RUN: opt -thinlto-bc -o %t %s +; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s +; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s + +; M0: @g = external constant [9 x i8*]{{$}} +; M1: @g = constant [9 x i8*] +@g = constant [9 x i8*] [ + i8* bitcast (i64 (i8*)* @ok1 to i8*), + i8* bitcast (i64 (i8*, i64)* @ok2 to i8*), + i8* bitcast (void (i8*)* @wrongtype1 to i8*), + i8* bitcast (i128 (i8*)* @wrongtype2 to i8*), + i8* bitcast (i64 ()* @wrongtype3 to i8*), + i8* bitcast (i64 (i8*, i8*)* @wrongtype4 to i8*), + i8* bitcast (i64 (i8*, i128)* @wrongtype5 to i8*), + i8* bitcast (i64 (i8*)* @usesthis to i8*), + i8* bitcast (i8 (i8*)* @reads to i8*) +], !type !0 + +; M0: define i64 @ok1 +; M1: define available_externally i64 @ok1 +define i64 @ok1(i8* %this) { + ret i64 42 +} + +; M0: define i64 @ok2 +; M1: define available_externally i64 @ok2 +define i64 @ok2(i8* %this, i64 %arg) { + ret i64 %arg +} + +; M0: define void @wrongtype1 +; M1: declare void @wrongtype1() +define void @wrongtype1(i8*) { + ret void +} + +; M0: define i128 @wrongtype2 +; M1: declare void @wrongtype2() +define i128 @wrongtype2(i8*) { + ret i128 0 +} + +; M0: define i64 @wrongtype3 +; M1: declare void @wrongtype3() +define i64 @wrongtype3() { + ret i64 0 +} + +; M0: define i64 @wrongtype4 +; M1: declare void @wrongtype4() +define i64 @wrongtype4(i8*, i8*) { + ret i64 0 +} + +; M0: define i64 @wrongtype5 +; M1: declare void @wrongtype5() +define i64 @wrongtype5(i8*, i128) { + ret i64 0 +} + +; M0: define i64 @usesthis +; M1: declare void @usesthis() +define i64 @usesthis(i8* %this) { + %i = ptrtoint i8* %this to i64 + ret i64 %i +} + +; M0: define i8 @reads +; M1: declare void @reads() +define i8 @reads(i8* %this) { + %l = load i8, i8* %this + ret i8 %l +} + +!0 = !{i32 0, !"typeid"}