Index: lib/Transforms/Instrumentation/GCOVProfiling.cpp =================================================================== --- lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -21,9 +21,9 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/UniqueVector.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/CFG.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/IRBuilder.h" @@ -98,7 +98,6 @@ // Get pointers to the functions in the runtime library. Constant *getStartFileFunc(); - Constant *getIncrementIndirectCounterFunc(); Constant *getEmitFunctionFunc(); Constant *getEmitArcsFunc(); Constant *getSummaryInfoFunc(); @@ -108,18 +107,11 @@ // pred block number for certain non-trivial edges. GlobalVariable *getEdgeStateValue(); - // Produce a table of pointers to counters, by predecessor and successor - // block number. - GlobalVariable *buildEdgeLookupTable(Function *F, GlobalVariable *Counter, - const UniqueVector &Preds, - const UniqueVector &Succs); - // Add the function to write out all our counters to the global destructor // list. Function * insertCounterWriteout(ArrayRef>); Function *insertFlush(ArrayRef>); - void insertIndirectCounterIncrement(); enum class GCovFileType { GCNO, GCDA }; std::string mangleName(const DICompileUnit *CU, GCovFileType FileType); @@ -639,7 +631,6 @@ if (!CU_Nodes) return false; bool Result = false; - bool InsertIndCounterIncrCode = false; for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { SmallVector, 8> CountersBySP; for (auto &F : M->functions()) { @@ -650,13 +641,23 @@ if (isUsingScopeBasedEH(F)) continue; if (!Result) Result = true; + DenseMap, unsigned> EdgeToCounter; unsigned Edges = 0; for (auto &BB : F) { TerminatorInst *TI = BB.getTerminator(); - if (isa(TI)) + if (isa(TI)) { + EdgeToCounter.insert( + std::make_pair(std::make_pair(&BB, nullptr), Edges)); ++Edges; - else - Edges += TI->getNumSuccessors(); + } else { + if (int successors = TI->getNumSuccessors()) { + for (int i = 0; i != successors; ++i) { + EdgeToCounter.insert(std::make_pair( + std::make_pair(&BB, TI->getSuccessor(i)), Edges)); + ++Edges; + } + } + } } ArrayType *CounterTy = @@ -668,63 +669,44 @@ "__llvm_gcov_ctr"); CountersBySP.push_back(std::make_pair(Counters, SP)); - UniqueVector ComplexEdgePreds; - UniqueVector ComplexEdgeSuccs; - - unsigned Edge = 0; + // If a BB has several predecessors, use a PHINode to select + // the correct counter. for (auto &BB : F) { - TerminatorInst *TI = BB.getTerminator(); - int Successors = isa(TI) ? 1 : TI->getNumSuccessors(); - if (Successors) { - if (Successors == 1) { - IRBuilder<> Builder(&*BB.getFirstInsertionPt()); - Value *Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0, - Edge); - Value *Count = Builder.CreateLoad(Counter); - Count = Builder.CreateAdd(Count, Builder.getInt64(1)); - Builder.CreateStore(Count, Counter); - } else if (BranchInst *BI = dyn_cast(TI)) { - IRBuilder<> Builder(BI); - Value *Sel = Builder.CreateSelect(BI->getCondition(), - Builder.getInt64(Edge), - Builder.getInt64(Edge + 1)); - Value *Counter = Builder.CreateInBoundsGEP( - Counters->getValueType(), Counters, {Builder.getInt64(0), Sel}); + const unsigned EdgeCount = + std::distance(pred_begin(&BB), pred_end(&BB)); + if (EdgeCount >= 1) { + IRBuilder<> Builder(&*BB.getFirstInsertionPt()); + Value *Counter; + + if (EdgeCount >= 2) { + Type *Int64PtrTy = Type::getInt64PtrTy(*Ctx); + PHINode *Phi = Builder.CreatePHI(Int64PtrTy, EdgeCount); + for (BasicBlock *Pred : predecessors(&BB)) { + const unsigned Edge = + EdgeToCounter.lookup(std::make_pair(Pred, &BB)); + Value *EdgeCounter = + Builder.CreateConstInBoundsGEP2_64(Counters, 0, Edge); + Phi->addIncoming(EdgeCounter, Pred); + } + Counter = Phi; + } else { + const unsigned Edge = EdgeToCounter.lookup( + std::make_pair(BB.getUniquePredecessor(), &BB)); + Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Edge); + } + Value *Count = Builder.CreateLoad(Counter); + Count = Builder.CreateAdd(Count, Builder.getInt64(1)); + Builder.CreateStore(Count, Counter); + + TerminatorInst *TI = BB.getTerminator(); + if (isa(TI)) { + const unsigned Edge = + EdgeToCounter.lookup(std::make_pair(&BB, nullptr)); + Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Edge); Value *Count = Builder.CreateLoad(Counter); Count = Builder.CreateAdd(Count, Builder.getInt64(1)); Builder.CreateStore(Count, Counter); - } else { - ComplexEdgePreds.insert(&BB); - for (int i = 0; i != Successors; ++i) - ComplexEdgeSuccs.insert(TI->getSuccessor(i)); } - - Edge += Successors; - } - } - - if (!ComplexEdgePreds.empty()) { - GlobalVariable *EdgeTable = - buildEdgeLookupTable(&F, Counters, - ComplexEdgePreds, ComplexEdgeSuccs); - GlobalVariable *EdgeState = getEdgeStateValue(); - - for (int i = 0, e = ComplexEdgePreds.size(); i != e; ++i) { - IRBuilder<> Builder(&*ComplexEdgePreds[i + 1]->getFirstInsertionPt()); - Builder.CreateStore(Builder.getInt32(i), EdgeState); - } - - for (int i = 0, e = ComplexEdgeSuccs.size(); i != e; ++i) { - // Call runtime to perform increment. - IRBuilder<> Builder(&*ComplexEdgeSuccs[i + 1]->getFirstInsertionPt()); - Value *CounterPtrArray = - Builder.CreateConstInBoundsGEP2_64(EdgeTable, 0, - i * ComplexEdgePreds.size()); - - // Build code to increment the counter. - InsertIndCounterIncrCode = true; - Builder.CreateCall(getIncrementIndirectCounterFunc(), - {EdgeState, CounterPtrArray}); } } } @@ -763,60 +745,9 @@ appendToGlobalCtors(*M, F, 0); } - if (InsertIndCounterIncrCode) - insertIndirectCounterIncrement(); - return Result; } -// All edges with successors that aren't branches are "complex", because it -// requires complex logic to pick which counter to update. -GlobalVariable *GCOVProfiler::buildEdgeLookupTable( - Function *F, - GlobalVariable *Counters, - const UniqueVector &Preds, - const UniqueVector &Succs) { - // TODO: support invoke, threads. We rely on the fact that nothing can modify - // the whole-Module pred edge# between the time we set it and the time we next - // read it. Threads and invoke make this untrue. - - // emit [(succs * preds) x i64*], logically [succ x [pred x i64*]]. - size_t TableSize = Succs.size() * Preds.size(); - Type *Int64PtrTy = Type::getInt64PtrTy(*Ctx); - ArrayType *EdgeTableTy = ArrayType::get(Int64PtrTy, TableSize); - - std::unique_ptr EdgeTable(new Constant *[TableSize]); - Constant *NullValue = Constant::getNullValue(Int64PtrTy); - for (size_t i = 0; i != TableSize; ++i) - EdgeTable[i] = NullValue; - - unsigned Edge = 0; - for (BasicBlock &BB : *F) { - TerminatorInst *TI = BB.getTerminator(); - int Successors = isa(TI) ? 1 : TI->getNumSuccessors(); - if (Successors > 1 && !isa(TI) && !isa(TI)) { - for (int i = 0; i != Successors; ++i) { - BasicBlock *Succ = TI->getSuccessor(i); - IRBuilder<> Builder(Succ); - Value *Counter = Builder.CreateConstInBoundsGEP2_64(Counters, 0, - Edge + i); - EdgeTable[((Succs.idFor(Succ) - 1) * Preds.size()) + - (Preds.idFor(&BB) - 1)] = cast(Counter); - } - } - Edge += Successors; - } - - GlobalVariable *EdgeTableGV = - new GlobalVariable( - *M, EdgeTableTy, true, GlobalValue::InternalLinkage, - ConstantArray::get(EdgeTableTy, - makeArrayRef(&EdgeTable[0],TableSize)), - "__llvm_gcda_edge_table"); - EdgeTableGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - return EdgeTableGV; -} - Constant *GCOVProfiler::getStartFileFunc() { Type *Args[] = { Type::getInt8PtrTy(*Ctx), // const char *orig_filename @@ -832,17 +763,6 @@ } -Constant *GCOVProfiler::getIncrementIndirectCounterFunc() { - Type *Int32Ty = Type::getInt32Ty(*Ctx); - Type *Int64Ty = Type::getInt64Ty(*Ctx); - Type *Args[] = { - Int32Ty->getPointerTo(), // uint32_t *predecessor - Int64Ty->getPointerTo()->getPointerTo() // uint64_t **counters - }; - FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false); - return M->getOrInsertFunction("__llvm_gcov_indirect_counter_increment", FTy); -} - Constant *GCOVProfiler::getEmitFunctionFunc() { Type *Args[] = { Type::getInt32Ty(*Ctx), // uint32_t ident @@ -886,19 +806,6 @@ return M->getOrInsertFunction("llvm_gcda_end_file", FTy); } -GlobalVariable *GCOVProfiler::getEdgeStateValue() { - GlobalVariable *GV = M->getGlobalVariable("__llvm_gcov_global_state_pred"); - if (!GV) { - GV = new GlobalVariable(*M, Type::getInt32Ty(*Ctx), false, - GlobalValue::InternalLinkage, - ConstantInt::get(Type::getInt32Ty(*Ctx), - 0xffffffff), - "__llvm_gcov_global_state_pred"); - GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - } - return GV; -} - Function *GCOVProfiler::insertCounterWriteout( ArrayRef > CountersBySP) { FunctionType *WriteoutFTy = FunctionType::get(Type::getVoidTy(*Ctx), false); @@ -1122,57 +1029,6 @@ return WriteoutF; } -void GCOVProfiler::insertIndirectCounterIncrement() { - Function *Fn = - cast(GCOVProfiler::getIncrementIndirectCounterFunc()); - Fn->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - Fn->setLinkage(GlobalValue::InternalLinkage); - Fn->addFnAttr(Attribute::NoInline); - if (Options.NoRedZone) - Fn->addFnAttr(Attribute::NoRedZone); - - // Create basic blocks for function. - BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", Fn); - IRBuilder<> Builder(BB); - - BasicBlock *PredNotNegOne = BasicBlock::Create(*Ctx, "", Fn); - BasicBlock *CounterEnd = BasicBlock::Create(*Ctx, "", Fn); - BasicBlock *Exit = BasicBlock::Create(*Ctx, "exit", Fn); - - // uint32_t pred = *predecessor; - // if (pred == 0xffffffff) return; - Argument *Arg = &*Fn->arg_begin(); - Arg->setName("predecessor"); - Value *Pred = Builder.CreateLoad(Arg, "pred"); - Value *Cond = Builder.CreateICmpEQ(Pred, Builder.getInt32(0xffffffff)); - BranchInst::Create(Exit, PredNotNegOne, Cond, BB); - - Builder.SetInsertPoint(PredNotNegOne); - - // uint64_t *counter = counters[pred]; - // if (!counter) return; - Value *ZExtPred = Builder.CreateZExt(Pred, Builder.getInt64Ty()); - Arg = &*std::next(Fn->arg_begin()); - Arg->setName("counters"); - Value *GEP = Builder.CreateGEP(Type::getInt64PtrTy(*Ctx), Arg, ZExtPred); - Value *Counter = Builder.CreateLoad(GEP, "counter"); - Cond = Builder.CreateICmpEQ(Counter, - Constant::getNullValue( - Builder.getInt64Ty()->getPointerTo())); - Builder.CreateCondBr(Cond, Exit, CounterEnd); - - // ++*counter; - Builder.SetInsertPoint(CounterEnd); - Value *Add = Builder.CreateAdd(Builder.CreateLoad(Counter), - Builder.getInt64(1)); - Builder.CreateStore(Add, Counter); - Builder.CreateBr(Exit); - - // Fill in the exit block. - Builder.SetInsertPoint(Exit); - Builder.CreateRetVoid(); -} - Function *GCOVProfiler:: insertFlush(ArrayRef > CountersBySP) { FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);